Upload
others
View
21
Download
0
Embed Size (px)
Citation preview
1.1
1.2
1.3
1.4
1.4.1
1.4.2
1.4.3
1.4.4
1.4.5
1.4.6
1.4.7
1.4.8
1.4.9
1.5
1.5.1
1.5.2
1.5.3
1.6
1.6.1
1.6.2
1.6.3
1.6.4
1.6.5
1.6.6
1.7
1.7.1
1.7.2
1.7.3
1.7.4
1.7.5
1.7.6
1.8
1.8.1
1.8.2
1.8.3
1.8.4
1.8.4.1
1.9
1.9.1
1.9.2
1.9.3
TableofContentsIntroduction
ModellingDocumentInheritance
AccessingShapesData
AQL
UsingJoinsinAQL
UsingDynamicAttributeNames
CreatingTest-datausingAQL
DiffingDocuments
AvoidingParameterInjection
MultilineQueryStrings
Migratingnamedgraphfunctionsto3.0
Migratinganonymousgraphfunctionsto3.0
Migratinggraphmeasurementsto3.0
Graph
FulldepthGraph-Traversal
UsingacustomVisitor
ExampleAQLQueriesforGraphs
UseCases/Examples
CrawlingGithubwithPromises
UsingArangoDBwithSails.js
PopulatingaTextbox
ExportingData
AccessingbasedocumentswithJava
AddXMLdatatoArangoDBwithJava
Administration
UsingAuthentication
ImportingData
ReplicatingData
XCopyInstallWindows
Migrating2.8to3.0
Showgrantsfunction
Compiling/Build
CompileonDebian
CompileonWindows
OpenSSL
RunningCustomBuild
Recompilingjemalloc
Cloud,DCOSandDocker
RunningonAWS
UpdateonAWS
RunningonAzure
1
1.9.4
1.9.5
1.9.6
1.9.7
1.9.8
1.10
1.10.1
1.10.2
1.10.3
1.10.4
DockerArangoDB
DockerwithNodeJSApp
IntheGiantSwarm
ArangoDBinMesos
DC/OS:Fullexample
Monitoring
Collectd-ReplicationSlaves
Collectd-Networkusage
Collectd-moreMetrics
Collectd-MonitoringFoxx
2
CookbookThiscookbookisfilledwithrecipestohelpyouunderstandthemulti-modeldatabaseArangoDBbetterandtohelpyouwithspecificproblems.
Youcanparticipateandwriteyourownrecipes.Youonlyneedtowritearecipeinmarkdownandmakeapullrequesttoourrepository.
Recipes
TherewillbesomesimplerecipestobringyouclosertoArangoDBandshowyoutheamountofpossibilitiesofourDatabase.TherealsowillbemorecomplexproblemstoshowyousolutiontospecificproblemsandthedepthofArangoDB.
Everyrecipeisdividedintothreeparts:
1. Problem:Adescriptionoftheproblem2. Solution:Adetailedsolutionofthegivenproblemwithcodeifanyisneeded3. Comment:Explanationofthesolution.Thispartisoptionaldependingonthecomplexityoftheproblem
Everyrecipehastagstoforabetteroverview:
#api,#aql,#arangosh,#collection,#database,#debian,#docker,#document,#driver,#foxx,#giantswarm,#graph,#howto,#java,#javascript,#join,#nodejs,#windows
Introduction
3
Modeldocumentinheritance
Problem
Howdoyoumodeldocumentinheritancegiventhatcollectionsdonotsupportthatfeature?
Solution
Letsassumeyouhavethreedocumentcollections:"subclass","class"and"superclass".Youalsohavetwoedgecollections:"sub_extends_class"and"class_extends_super".
Youcancreatethemviaarangoshorfoxx:
vargraph_module=require("com/arangodb/general-graph");
varg=graph_module._create("inheritance");
g._extendEdgeDefinitions(graph_module._directedRelation("sub_extends_class",["subclass"],["class"]));
g._extendEdgeDefinitions(graph_module._directedRelation("class_extends_super",["class"],["superclass"]));
Thismakessurewhenusingthegraphinterfacethattheinheritancelookslike:
sub→classclass→supersuper→sub
Tomakesureeverythingworksasexpectedyoushouldusethebuilt-intraversalincombinationwithFoxx.Thisallowsyoutoaddtheinheritancesecuritylayereasily.Tousetraversalsinfoxxsimplyaddthefollowinglinebeforedefiningroutes:
vartraversal=require("org/arangodb/graph/traversal");
varTraverser=traversal.Traverser;
AlsoyoucanaddthefollowingendpointinFoxx:
varreaderConfig={
datasource:traversal.graphDatasourceFactory("inheritance"),
expander:traversal.outboundExpander,//Goupwardsinthetree
visitor:function(config,result,vertex,path){
for(keyinvertex){
if(vertex.hasOwnProperty(key)&&!result.hasOwnProperty(key)){
result[key]=vertex[key]//Storeonlyattributesthathavenotyetbeenfound
}
}
}
};
controller.get("load/:collection/:key",function(req,res){
varresult={};
varid=res.params("collection")+"/"+res.params("key");
vartraverser=newTraverser(readerConfig);
traverser.traverse(result,g.getVertex(id));
res.json(result);
});
Thiswillmakesuretoiteratethecompleteinheritancetreeupwardstotherootelementandwillreturnallvaluesonthepathwerethefirstinstanceofthisvalueiskept
Comment
Youshouldgowithedgesbecauseitismucheasiertoquerythemifyouhaveatheoreticallyunlimiteddepthininheritance.IfyouhaveafixedinheritancedepthyoucouldalsogowithanattributeinthedocumentreferencingtheparentandexecutejoinsinAQL.
ModellingDocumentInheritance
4
Author:MichaelHackstein
Tags:#graph#document
ModellingDocumentInheritance
5
AccessingShapesData
Problem
Documentsinacollectionmayhavedifferentshapesassociatedwiththem.Thereisnowaytoquerytheshapesdatadirectly.Sohowdoyousolvethisproblem?
Solution
Therearetwopossiblewaystodothis.
A)Thefastwaywithsomerandomsamplings:
1. Askforarandomdocument(db.<collection>.any())andnoteitstop-levelattributenames2. Repeatthisforatleast10times.Afterthatrepeatitonlyifyouthinkit'sworthit.
Followingisanexampleofanimplementation:
attributes(db.myCollection);
functionattributes(collection){
"usestrict"
varprobes=10;
varmaxRounds=3;
varthreshold=0.5;
varmaxDocuments=collection.count();
if(maxDocuments<probes){
probes=maxDocuments;
}
if(probes===0){
return[];
}
varattributes={};
while(maxRounds--){
varnewDocuments=0;
varn=probes;
while(n--){
vardoc=collection.any();
varfound=false;
varkeys=Object.keys(doc);
for(vari=0;i<keys.length;++i){
if(attributes.hasOwnProperty(keys[i])){
++attributes[keys[i]];
}
else{
attributes[keys[i]]=1;
found=true;
}
}
if(found){
++newDocuments;
}
}
if(newDocuments/probes<=threshold){
break;
}
}
AccessingShapesData
6
returnObject.keys(attributes);
}
B)Thewaytofindalltop-levelattributes
Ifyoudon'tmindtomakesomeextrainsertsandyoudon'tcareaboutdeletionorupdatesofdocumentsyoucanusethefollowing:
db._create("mykeys");
db.mykeys.ensureUniqueSkiplist("attribute");
functioninsert(collection,document){
varresult=collection.save(document);
try{
varkeys=Objects.keys(document);
for(i=0;i<keys.length;++i){
try{
db.mykeys.save({attribute:keys[i]});
}
catch(err1){
//potentialuniquekeyconstraintviolations
}
}
}
catch(err2){
}
returnresult;
}
CommentA)Thefastwaywithsomerandomsamplings:
Yougetsomerandomsamplingwithboundedcomplexity.Ifyouhaveavarietyofattributesyoushouldrepeattheproceduremorethan10times.
Theprocedurecanbeimplementedasaserversideaction.
B)Thewaytofindalltop-levelattributes:
Thisprocedurewillnotcareaboutupdatesordeletionsofdocuments.Alsoonlythetop-levelattributeofthedocumentswillbeinsertedandnestedoneignored.
Theprocedurecanbeimplementedasaserversideaction.
Author:Arangodb
Tags:#collection#database
AccessingShapesData
7
AQL
UsingAQLingeneral
UsingJoinsinAQL
UsingDynamicAttributeNames
CreatingTest-datausingAQL
DiffingDocuments
AvoidingParameterInjection
MultilineQueryStrings
Migratingfrom2.xto3.0
Migratingnamedgraphfunctionsto3.0
Migratinganonymousgraphfunctionsto3.0
Migratinggraphmeasurementsto3.0
AQL
8
UsingJoinsinAQL
Problem
IwanttojoindocumentsfromcollectionsinanAQLquery.
One-to-Many:Ihaveacollectionusersandacollectioncities.AuserlivesinacityandIneedthecityinformationduringthequery.
Many-To-Many:Ihaveacollectionauthorsandbooks.Anauthorcanwritemanybooksandabookcanhavemanyauthors.Iwanttoreturnalistofbookswiththeirauthors.ThereforeIneedtojointheauthorsandbooks.
Solution
UnlikemanyNoSQLdatabases,ArangoDBdoessupportjoinsinAQLqueries.Thisissimilartothewaytraditionalrelationaldatabaseshandlethis.However,becausedocumentsallowformoreflexibility,joinsarealsomoreflexible.Thefollowingsectionsprovidesolutionsforcommonquestions.
One-To-Many
Youhaveacollectioncalledusers.Usersliveincityandacityisidentifiedbyitsprimarykey.Inprincipleyoucanembeddedthecitydocumentintotheusersdocumentandbehappywithit.
{
"_id":"users/2151975421",
"_key":"2151975421",
"_rev":"2151975421",
"name":{
"first":"John",
"last":"Doe"
},
"city":{
"name":"Metropolis"
}
}
Thisworkswellformanyusecases.Nowassume,thatyouhaveadditionalinformationaboutthecity,likethenumberofpeoplelivinginit.Itwouldbeimpracticaltochangeeachandeveryuserdocumentifthisnumberschanges.Thereforeitisgoodideatoholdthecityinformationinaseparatecollection.
arangosh>db.cities.document("cities/2241300989");
{
"population":1000,
"name":"Metropolis",
"_id":"cities/2241300989",
"_rev":"2241300989",
"_key":"2241300989"
}
Nowyouinsteadofembeddingthecitydirectlyintheuserdocument,youcanusethekeyofthecity.
arangosh>db.users.document("users/2290649597");
{
"name":{
"first":"John",
"last":"Doe"
},
"city":"cities/2241300989",
"_id":"users/2290649597",
"_rev":"2290649597",
"_key":"2290649597"
}
UsingJoinsinAQL
9
Wecannowjointhesetwocollectionsveryeasily.
arangosh>db._query(
........>"FORuINusers"+
........>"FORcINcities"+
........>"FILTERu.city==c._idRETURN{user:u,city:c}"
........>).toArray()
[
{
"user":{
"name":{
"first":"John",
"last":"Doe"
},
"city":"cities/2241300989",
"_id":"users/2290649597",
"_rev":"2290649597",
"_key":"2290649597"
},
"city":{
"population":1000,
"name":"Metropolis",
"_id":"cities/2241300989",
"_rev":"2241300989",
"_key":"2241300989"
}
}
]
UnlikeSQLthereisnospecialJOINkeyword.Theoptimizerensuresthattheprimaryindexisusedintheabovequery.
However,veryoftenitismuchmoreconvenientfortheclientofthequeryifasingledocumentwouldbereturned,wherethecityinformationisembeddedintheuserdocument-asinthesimpleexampleabove.WithAQLthereyoudonotneedtoforgothissimplification.
arangosh>db._query(
........>"FORuINusers"+
........>"FORcINcities"+
........>"FILTERu.city==c._idRETURNmerge(u,{city:c})"
........>).toArray()
[
{
"_id":"users/2290649597",
"_key":"2290649597",
"_rev":"2290649597",
"name":{
"first":"John",
"last":"Doe"
},
"city":{
"_id":"cities/2241300989",
"_key":"2241300989",
"_rev":"2241300989",
"population":1000,
"name":"Metropolis"
}
}
]
Soyoucanhaveboth:theconvenientrepresentationoftheresultforyourclientandtheflexibilityofjoinsforyourdatamodel.
Many-To-Many
Intherelationalwordyouneedathirdtabletomodelthemany-to-manyrelation.InArangoDByouhaveachoicedependingontheinformationyouaregoingtostoreandthetypeofquestionsyouaregoingtoask.
Assumethatauthorsarestoredinonecollectionandbooksinasecond.Ifallyouneedis"whicharetheauthorsofabook"thenyoucaneasilymodelthisasalistattributeinusers.
UsingJoinsinAQL
10
Ifyouwanttostoremoreinformation,forexamplewhichauthorwrotewhichpageinaconferenceproceeding,orifyoualsowanttoknow"whichbookswerewrittenbywhichauthor",youcanuseedgecollections.Thisisverysimilartothe"jointable"fromtherelationalworld.
EmbeddedLists
Ifyouonlywanttostoretheauthorsofabook,youcanembedthemaslistinthebookdocument.Thereisnoneedforaseparatecollection.
arangosh>db.authors.toArray()
[
{
"_id":"authors/2661190141",
"_key":"2661190141",
"_rev":"2661190141",
"name":{
"first":"Maxima",
"last":"Musterfrau"
}
},
{
"_id":"authors/2658437629",
"_key":"2658437629",
"_rev":"2658437629",
"name":{
"first":"John",
"last":"Doe"
}
}
]
Youcanquerybooks
arangosh>db._query("FORbINbooksRETURNb").toArray();
[
{
"_id":"books/2681506301",
"_key":"2681506301",
"_rev":"2681506301",
"title":"ThebeautyofJOINS",
"authors":[
"authors/2661190141",
"authors/2658437629"
]
}
]
andjointheauthorsinaverysimilarmannergivenintheone-to-manysection.
arangosh>db._query(
........>"FORbINbooks"+
........>"LETa=(FORxINb.authors"+
........>"FORaINauthorsFILTERx==a._idRETURNa)"+
........>"RETURN{book:b,authors:a}"
........>).toArray();
[
{
"book":{
"title":"ThebeautyofJOINS",
"authors":[
"authors/2661190141",
"authors/2658437629"
],
"_id":"books/2681506301",
"_rev":"2681506301",
"_key":"2681506301"
},
"authors":[
{
"name":{
UsingJoinsinAQL
11
"first":"Maxima",
"last":"Musterfrau"
},
"_id":"authors/2661190141",
"_rev":"2661190141",
"_key":"2661190141"
},
{
"name":{
"first":"John",
"last":"Doe"
},
"_id":"authors/2658437629",
"_rev":"2658437629",
"_key":"2658437629"
}
]
}
]
orembedtheauthorsdirectly
arangosh>db._query(
........>"FORbINbooksLETa=("+
........>"FORxINb.authors"+
........>"FORaINauthorsFILTERx==a._idRETURNa)"+
........>"RETURNmerge(b,{authors:a})"
........>).toArray();
[
{
"_id":"books/2681506301",
"_key":"2681506301",
"_rev":"2681506301",
"title":"ThebeautyofJOINS",
"authors":[
{
"_id":"authors/2661190141",
"_key":"2661190141",
"_rev":"2661190141",
"name":{
"first":"Maxima",
"last":"Musterfrau"
}
},
{
"_id":"authors/2658437629",
"_key":"2658437629",
"_rev":"2658437629",
"name":{
"first":"John",
"last":"Doe"
}
}
]
}
]
UsingEdgeCollections
Ifyoualsowanttoquerywhichbooksarewrittenbyagivenauthor,embeddingauthorsinthebookdocumentispossible,butitismoreefficienttouseaedgecollectionsforspeed.
Oryouarepublishingaproceeding,thenyouwanttostorethepagestheauthorhaswrittenaswell.Thisinformationcanbestoredintheedgedocument.
Firstcreatetheusers
arangosh>db._create("authors");
[ArangoCollection2926807549,"authors"(typedocument,statusloaded)]
arangosh>db.authors.save({name:{first:"John",last:"Doe"}})
UsingJoinsinAQL
12
{
"error":false,
"_id":"authors/2935261693",
"_rev":"2935261693",
"_key":"2935261693"
}
arangosh>db.authors.save({name:{first:"Maxima",last:"Musterfrau"}})
{
"error":false,
"_id":"authors/2938210813",
"_rev":"2938210813",
"_key":"2938210813"
}
Nowcreatethebookswithoutanyauthorinformation.
arangosh>db._create("books");
[ArangoCollection2928380413,"books"(typedocument,statusloaded)]
arangosh>db.books.save({title:"ThebeautyofJOINS"});
{
"error":false,
"_id":"books/2980088317",
"_rev":"2980088317",
"_key":"2980088317"
}
Anedgecollectionisnowusedtolinkauthorsandbooks.
arangosh>db._createEdgeCollection("written");
[ArangoCollection2931132925,"written"(typeedge,statusloaded)]
arangosh>db.written.save("authors/2935261693",
........>"books/2980088317",
........>{pages:"1-10"})
{
"error":false,
"_id":"written/3006237181",
"_rev":"3006237181",
"_key":"3006237181"
}
arangosh>db.written.save("authors/2938210813",
........>"books/2980088317",
........>{pages:"11-20"})
{
"error":false,
"_id":"written/3012856317",
"_rev":"3012856317",
"_key":"3012856317"
}
InordertogetallbookswiththeirauthorsyoucanuseNEIGHBORS.
arangosh>db._query(
........>"FORbINbooksRETURN"+
........>"{"+
........>"book:b,"+
........>"authors:NEIGHBORS(books,"+
........>"written,"+
........>"b._id,"+
........>"'inbound'"+
........>")}"
........>).toArray();
[
{
"book":{
"_id":"books/2980088317",
"_rev":"2980088317",
"_key":"2980088317",
"title":"ThebeautyofJOINS"
UsingJoinsinAQL
13
},
"authors":[
{
"edge":{
"_id":"written/3006237181",
"_from":"authors/2935261693",
"_to":"books/2980088317",
"_rev":"3006237181",
"_key":"3006237181",
"pages":"1-10"
},
"vertex":{
"_id":"authors/2935261693",
"_rev":"2935261693",
"_key":"2935261693",
"name":{
"first":"John",
"last":"Doe"
}
}
},
{
"edge":{
"_id":"written/3012856317",
"_from":"authors/2938210813",
"_to":"books/2980088317",
"_rev":"3012856317",
"_key":"3012856317",
"pages":"11-20"
},
"vertex":{
"_id":"authors/2938210813",
"_rev":"2938210813",
"_key":"2938210813",
"name":{
"first":"Maxima",
"last":"Musterfrau"
}
}
}
]
}
]
Orifyouwanttohidetheinformationstoredintheedge.
arangosh>db._query(
........>"FORbINbooksRETURN{"+
........>"book:b,authors:"+
........>"NEIGHBORS(books,written,b._id,'inbound')[*].vertex}"
........>).toArray();
[
{
"book":{
"title":"ThebeautyofJOINS",
"_id":"books/2980088317",
"_rev":"2980088317",
"_key":"2980088317"
},
"authors":[
{
"_id":"authors/2935261693",
"_rev":"2935261693",
"_key":"2935261693",
"name":{
"first":"John",
"last":"Doe"
}
},
{
"_id":"authors/2938210813",
"_rev":"2938210813",
"_key":"2938210813",
"name":{
"first":"Maxima",
UsingJoinsinAQL
14
"last":"Musterfrau"
}
}
]
}
]
Oragainembedtheauthorsdirectlyintothebookdocument.
arangosh>db._query(
........>"FORbINbooksRETURNmerge("+
........>"b,"+
........>"{"+
........>"authors:"+
........>"NEIGHBORS(books,written,b._id,'inbound')[*].vertex})"
........>).toArray();
[
{
"_id":"books/2980088317",
"_rev":"2980088317",
"_key":"2980088317",
"title":"ThebeautyofJOINS",
"authors":[
{
"_id":"authors/2935261693",
"_rev":"2935261693",
"_key":"2935261693",
"name":{
"first":"John",
"last":"Doe"
}
},
{
"_id":"authors/2938210813",
"_rev":"2938210813",
"_key":"2938210813",
"name":{
"first":"Maxima",
"last":"Musterfrau"
}
}
]
}
]
Ifyouneedtheauthorsandtheirbooks,simplyreversethedirection.
arangosh>db._query(
........>"FORaINauthorsRETURN"+
........>"merge(a,"+
........>"{books:NEIGHBORS(authors,written,a._id,'outbound')[*].vertex})"
........>).toArray();
[
{
"_id":"authors/2938210813",
"_rev":"2938210813",
"_key":"2938210813",
"name":{
"first":"Maxima",
"last":"Musterfrau"
},
"books":[
{
"_id":"books/2980088317",
"_rev":"2980088317",
"_key":"2980088317",
"title":"ThebeautyofJOINS"
}
]
},
{
"_id":"authors/2935261693",
"_rev":"2935261693",
UsingJoinsinAQL
15
"_key":"2935261693",
"name":{
"first":"John",
"last":"Doe"
},
"books":[
{
"_id":"books/2980088317",
"_rev":"2980088317",
"_key":"2980088317",
"title":"ThebeautyofJOINS"
}
]
}
]
Authors:FrankCeller
Tags:#join#aql
UsingJoinsinAQL
16
UsingdynamicattributenamesinAQL
Problem
IwantanAQLquerytoreturnresultswithattributenamesassembledbyafunction,orwithavariablenumberofattributes.
Thiswillnotworkbyspecifyingtheresultusingaregularobjectliteral,asobjectliteralsrequirethenamesandnumbersofattributestobefixedatquerycompiletime.
SolutionThereareseveralsolutionstogettingdynamicattributenamestowork.
Subquerysolution
Ageneralsolutionistoletasubqueryoranotherfunctiontoproducethedynamicattributenames,andfinallypassthemthroughtheZIP()functiontocreateanobjectfromthem.
Let'sassumewewanttoprocessthefollowinginputdocuments:
{"name":"test","gender":"f","status":"active","type":"user"}
{"name":"dummy","gender":"m","status":"inactive","type":"unknown","magicFlag":23}
Let'salsoassumeourgoalforeachofthesedocumentsistoreturnonlytheattributenamesthatcontainthelettera,togetherwiththeirrespectivevalues.
Toextracttheattributenamesandvaluesfromtheoriginaldocuments,wecanuseasubqueryasfollows:
LETdocuments=[
{"name":"test","gender":"f","status":"active","type":"user"},
{"name":"dummy","gender":"m","status":"inactive","type":"unknown","magicFlag":23}
]
FORdocINdocuments
RETURN(
FORnameINATTRIBUTES(doc)
FILTERLIKE(name,'%a%')
RETURN{
name:name,
value:doc[name]
}
)
Thesubquerywillonlyletattributenamespassthatcontainthelettera.Theresultsofthesubqueryarethenmadeavailabletothemainqueryandwillbereturned.Buttheattributenamesintheresultarestillnameandvalue,sowe'renotthereyet.
Solet'salsoemployAQL'sZIP()function,whichcancreateanobjectfromtwoarrays:
thefirstparametertoZIP()isanarraywiththeattributenamesthesecondparametertoZIP()isanarraywiththeattributevalues
Insteadofdirectlyreturningthesubqueryresult,wefirstcaptureitinavariable,andpassthevariable'snameandvaluecomponentsintoZIP()likethis:
LETdocuments=[
{"name":"test","gender":"f","status":"active","type":"user"},
{"name":"dummy","gender":"m","status":"inactive","type":"unknown","magicFlag":23}
]
FORdocINdocuments
LETattributes=(
UsingDynamicAttributeNames
17
FORnameINATTRIBUTES(doc)
FILTERLIKE(name,'%a%')
RETURN{
name:name,
value:doc[name]
}
)
RETURNZIP(attributes[*].name,attributes[*].value)
Notethatwehavetousetheexpansionoperator([*])onattributesbecauseattributesitselfisanarray,andwewanteitherthenameattributeorthevalueattributeofeachofitsmembers.
Toprovethisisworking,hereistheabovequery'sresult:
[
{
"name":"test",
"status":"active"
},
{
"name":"dummy",
"status":"inactive",
"magicFlag":23
}
]
Ascanbeseen,thetworesultshaveadifferentamountofresultattributes.Wecanalsomaketheresultabitmoredynamicbyprefixingeachattributewiththevalueofthenameattribute:
LETdocuments=[
{"name":"test","gender":"f","status":"active","type":"user"},
{"name":"dummy","gender":"m","status":"inactive","type":"unknown","magicFlag":23}
]
FORdocINdocuments
LETattributes=(
FORnameINATTRIBUTES(doc)
FILTERLIKE(name,'%a%')
RETURN{
name:CONCAT(doc.name,'-',name),
value:doc[name]
}
)
RETURNZIP(attributes[*].name,attributes[*].value)
Thatwillgiveusdocument-specificattributenameslikethis:
[
{
"test-name":"test",
"test-status":"active"
},
{
"dummy-name":"dummy",
"dummy-status":"inactive",
"dummy-magicFlag":23
}
]
Usingexpressionsasattributenames(ArangoDB2.5)
Ifthenumberofdynamicattributestoreturnisknowninadvance,andonlytheattributenamesneedtobecalculatedusinganexpression,thenthereisanothersolution.
ArangoDB2.5andhigherallowusingexpressionsinsteadoffixedattributenamesinobjectliterals.Usingexpressionsasattributenamesrequiresenclosingtheexpressioninextra[and]todisambiguatethemfromregular,unquotedattributenames.
UsingDynamicAttributeNames
18
Let'screatearesultthatreturnstheoriginaldocumentdatacontainedinadynamicallynamedattribute.We'llbeusingtheexpressiondoc.typefortheattributename.We'llalsoreturnsomeotherattributesfromtheoriginaldocuments,butprefixthemwiththedocuments'_keyattributevalues.Forthiswealsoneedattributenameexpressions.
Hereisaqueryshowinghowtodothis.Theattributenameexpressionsallrequiredtobeenclosedin[and]inordertomakethiswork:
LETdocuments=[
{"_key":"3231748397810","gender":"f","status":"active","type":"user"},
{"_key":"3231754427122","gender":"m","status":"inactive","type":"unknown"}
]
FORdocINdocuments
RETURN{
[doc.type]:{
[CONCAT(doc._key,"_gender")]:doc.gender,
[CONCAT(doc._key,"_status")]:doc.status
}
}
Thiswillreturn:
[
{
"user":{
"3231748397810_gender":"f",
"3231748397810_status":"active"
}
},
{
"unknown":{
"3231754427122_gender":"m",
"3231754427122_status":"inactive"
}
}
]
Note:attributenameexpressionsandregular,unquotedattributenamescanbemixed.
Author:JanSteemann
Tags:#aql
UsingDynamicAttributeNames
19
CreatingtestdatawithAQL
Problem
Iwanttocreatesometestdocuments.
Solution
Ifyouhaven'tyetcreatedacollectiontoholdthedocuments,createonenowusingtheArangoShell:
db._create("myCollection");
ThishascreatedacollectionnamedmyCollection.
OneoftheeasiestwaystofillacollectionwithtestdataistouseanAQLquerythatiteratesoverarange.
RunthefollowingAQLqueryfromtheAQLeditorinthewebinterfacetoinsert1,000documentsintothejustcreatedcollection:
FORiIN1..1000
INSERT{name:CONCAT("test",i)}INmyCollection
Thenumberofdocumentstocreatecanbemodifiedeasilybeadjustingtherangeboundaryvalues.
Tocreatemorecomplextestdata,adjusttheAQLquery!
Let'ssaywealsowantastatusattribute,andfillitwithintegervaluesbetween1to(including)5,withequaldistribution.Agoodwaytoachievethisistousethemodulooperator(%):
FORiIN1..1000
INSERT{
name:CONCAT("test",i),
status:1+(i%5)
}INmyCollection
Tocreatepseudo-randomvalues,usetheRAND()function.Itcreatespseudo-randomnumbersbetween0and1.Usesomefactortoscaletherandomnumbers,andFLOOR()toconvertthescalednumberbacktoaninteger.
Forexample,thefollowingquerypopulatesthevalueattributewithnumbersbetween100and150(including):
FORiIN1..1000
INSERT{
name:CONCAT("test",i),
value:100+FLOOR(RAND()*(150-100+1))
}INmyCollection
Afterthetestdatahasbeencreated,itisoftenhelpfultoverifyit.TheRAND()functionisalsoagoodcandidateforretrievingarandomsampleofthedocumentsinthecollection.Thisquerywillretrieve10randomdocuments:
FORdocINmyCollection
SORTRAND()
LIMIT10
RETURNdoc
TheCOLLECTclauseisaneasymechanismtorunanaggregateanalysisonsomeattribute.Let'ssaywewantedtoverifythedatadistributioninsidethestatusattribute.Inthiscasewecouldrun:
FORdocINmyCollection
COLLECTvalue=doc.valueWITHCOUNTINTOcount
RETURN{
CreatingTest-datausingAQL
20
value:value,
count:count
}
Theabovequerywillprovidethenumberofdocumentsperdistinctvalue.
Author:JanSteemann
Tags:#aql
CreatingTest-datausingAQL
21
DiffingTwoDocumentsinAQL
Problem
HowtocreateadiffofdocumentsinAQL
Solution
Thoughthereisnobuilt-inAQLfunctiontodifftwodocuments,itiseasilypossibletobuildyourownlikeinthefollowingquery:
/*inputdocument1*/
LETdoc1={
"foo":"bar",
"a":1,
"b":2
}
/*inputdocument2*/
LETdoc2={
"foo":"baz",
"a":2,
"c":3
}
/*collectattributespresentindoc1,butmissingindoc2*/
LETmissing=(
FORkeyINATTRIBUTES(doc1)
FILTER!HAS(doc2,key)
RETURN{
[key]:doc1[key]
}
)
/*collectattributespresentinbothdocs,butthathavedifferentvalues*/
LETchanged=(
FORkeyINATTRIBUTES(doc1)
FILTERHAS(doc2,key)&&doc1[key]!=doc2[key]
RETURN{
[key]:{
old:doc1[key],
new:doc2[key]
}
}
)
/*collectattributespresentindoc2,butmissingindoc1*/
LETadded=(
FORkeyINATTRIBUTES(doc2)
FILTER!HAS(doc1,key)
RETURN{
[key]:doc2[key]
}
)
/*returnfinalresult*/
RETURN{
"missing":missing,
"changed":changed,
"added":added
}
Note:Thequerymaylookabitlengthy,butmuchofthatisduetoformatting.Amoreterseversioncanbefoundbelow.
Theabovequerywillreturnadocumentwiththreeattributes:
missing:Containsallattributesonlypresentinfirstdocument(i.e.missinginseconddocument)
DiffingDocuments
22
changed:Containsallattributespresentinbothdocumentsthathavedifferentvaluesadded:Containsallattributesonlypresentinseconddocument(i.e.missinginfirstdocument)
Forthetwoexampledocumentsitwillreturn:
[
{
"missing":[
{
"b":2
}
],
"changed":[
{
"foo":{
"old":"bar",
"new":"baz"
}
},
{
"a":{
"old":1,
"new":2
}
}
],
"added":[
{
"c":3
}
]
}
]
Thatoutputformatwasthefirstthatcametomymind.Itisofcoursepossibletoadjustthequerysoitproducesadifferentoutputformat.
FollowingisaversionofthesamequerythatcanbeinvokedfromJavaScripteasily.Itpassesthetwodocumentsasbindparametersandcallsdb._query.Thequeryisnowanone-liner(lessreadablebuteasiertocopy&paste):
bindVariables={
doc1:{"foo":"bar","a":1,"b":2},
doc2:{"foo":"baz","a":2,"c":3}
};
query="LETdoc1=@doc1,doc2=@doc2,missing=(FORkeyINATTRIBUTES(doc1)FILTER!HAS(doc2,key)RETURN{[key]:doc1[
key]}),changed=(FORkeyINATTRIBUTES(doc1)FILTERHAS(doc2,key)&&doc1[key]!=doc2[key]RETURN{[key]:{old:doc1[
key],new:doc2[key]}}),added=(FORkeyINATTRIBUTES(doc2)FILTER!HAS(doc1,key)RETURN{[key]:doc2[key]})RETURN
{missing:missing,changed:changed,added:added}";
result=db._query(query,bindVariables).toArray();
Author:JanSteemann
Tags:#howto#aql
DiffingDocuments
23
AvoidingparameterinjectioninAQL
Problem
Idon'twantmyAQLqueriestobeaffectedbyparameterinjection.
Whatisparameterinjection?
Parameterinjectionmeansthatpotentiallycontentisinsertedintoaquery,andthatinjectionmaychangethemeaningofthequery.Itisasecurityissuethatmayallowanattackertoexecutearbitraryqueriesonthedatabasedata.
Itoftenoccursifapplicationstrustfullyinsertuser-providedinputsintoaquerystring,anddonotfullyorincorrectlyfilterthem.Italsooccursoftenwhenapplicationsbuildqueriesnaively,withoutusingsecuritymechanismsoftenprovidedbydatabasesoftwareorqueryingmechanisms.
ParameterinjectionexamplesAssemblingquerystringswithsimplestringconcatenationlookstrivial,butispotentiallyunsafe.Let'sstartwithasimplequerythat'sfedwithsomedynamicinputvalue,let'ssayfromawebform.AclientapplicationoraFoxxroutehappilypicksuptheinputvalue,andputsitintoaquery:
/*evil!*/
varwhat=req.params("searchValue");/*userinputvaluefromwebform*/
...
varquery="FORdocINcollectionFILTERdoc.value=="+what+"RETURNdoc";
db._query(query,params).toArray();
Theabovewillprobablyworkfinefornumericinputvalues.
Whatcouldanattackerdotothisquery?HereareafewsuggestionstouseforthesearchValueparameter:
forreturningalldocumentsinthecollection:1||trueforremovingalldocuments:1||trueREMOVEdocINcollection//forinsertingnewdocuments:1||trueINSERT{foo:"bar"}INcollection//
Itshouldhavebecomeobviousthatthisisextremelyunsafeandshouldbeavoided.
Anpatternoftenseentocounteractthisistryingtoquoteandescapepotentiallyunsafeinputvaluesbeforeputtingthemintoquerystrings.Thismayworkinsomesituations,butit'seasytooverlooksomethingorgetitsubtlywrong:
/*we'resanitzingnow,butit'sstillevil!*/
varvalue=req.params("searchValue").replace(/'/g,'');
...
varquery="FORdocINcollectionFILTERdoc.value=='"+value+"'RETURNdoc";
db._query(query,params).toArray();
Theaboveexampleusessinglequotesforenclosingthepotentiallyunsafeuserinput,andalsoreplacesallsinglequotesintheinputvaluebeforehand.Notonlymaythatchangetheuserinput(leadingtosubtleerrorssuchas"whydoesmysearchforO'Briendon'treturnanyresults?"),butitisalsounsafe.Iftheuserinputcontainsabackslashattheend(e.g.foobar\),thatbackslashwillescapetheclosingsinglequote,allowingtheuserinputtobreakoutofthestringfenceagain.
Itgetsworseifuserinputisinsertedintothequeryatmultipleplaces.Let'sassumewehaveaquerywithtwodynamicvalues:
query="FORdocINcollectionFILTERdoc.value=='"+value+"'&&doc.type=='"+type+"'RETURNdoc";
Ifanattackerinserted\forparametervalueand||trueREMOVEdocINcollection//forparametertype,thentheeffectivequerywouldbecome
AvoidingParameterInjection
24
FORdocINcollectionFILTERdoc.value=='\'&&doc.type=='||trueREMOVEdocINcollection//'RETURNdoc
whichishighlyundesirable.
Solution
Insteadofmixingquerystringfragmentswithuserinputsnaivelyviastringconcatenation,useeitherbindparametersoraquerybuilder.Bothcanhelptoavoidtheproblemofinjection,becausetheyallowseparatingtheactualqueryoperations(likeFOR,INSERT,REMOVE)from(userinput)values.
Thisrecipefocusesonusingbindparameters.Thisisnottosaythatquerybuildersshouldn'tbeused.Theyweresimplyomittedhereforthesakeofsimplicity.TogetstartedwithausinganAQLquerybuilderinArangoDBorotherJavaScriptenvironments,havealookataqb(whichcomesbundledwithArangoDB).InsideArangoDB,therearealsoFoxxquerieswhichcanbecombinedwithaqb.
Whatbindparametersare
BindparametersinAQLqueriesarespecialtokensthatactasplaceholdersforactualvalues.Here'sanexample:
FORdocINcollection
FILTERdoc.value==@what
RETURNdoc
Intheabovequery,@whatisabindparameter.Inordertoexecutethisquery,avalueforbindparameter@whatmustbespecified.Otherwisequeryexecutionwillfailwitherror1551(novaluespecifiedfordeclaredbindparameter).Ifavaluefor@whatgetsspecified,thequerycanbeexecuted.However,thequerystringandthebindparametervalues(i.e.thecontentsofthe@whatbindparameter)willbehandledseparately.What'sinthebindparameterwillalwaysbetreatedasavalue,anditcan'tgetoutofitssandboxandchangethesemanticmeaningofaquery.
Howbindparametersareused
Toexecuteaquerywithbindparameters,thequerystring(containingthebindparameters)andthebindparametervaluesarespecifiedseparately(notethatwhenthebindparametervalueisassigned,theprefix@needstobeomitted):
/*querystringwithbindparameter*/
varquery="FORdocINcollectionFILTERdoc.value==@whatRETURNdoc";
/*actualvalueforbindparameter*/
varparams={what:42};
/*runquery,specifyingquerystringandbindparameterseparately*/
db._query(query,params).toArray();
Ifamalicioususerwouldset@whattoavalueof1||true,thiswouldn'tdoanyharm.AQLwouldtreatthecontentsof@whatasasinglestringtoken,andthemeaningofthequerywouldremainunchanged.Theactuallyexecutedquerywouldbe:
FORdocINcollection
FILTERdoc.value=="1||true"
RETURNdoc
Thankstobindparametersitisalsoimpossibletoturnaselection(i.e.read-only)queryintoadatadeletionquery.
UsingJavaScriptvariablesasbindparameters
Thereisalsoatemplatestringgeneratorfunctionaqlthatcanbeusedtosafely(andconveniently)builtAQLqueriesusingJavaScriptvariablesandexpressions.Itcanbeinvokedasfollows:
constaql=require('@arangodb')aql;//notneededinarangosh
varvalue="someinputvalue";
AvoidingParameterInjection
25
varquery=aql`FORdocINcollection
FILTERdoc.value==${value}
RETURNdoc`;
varresult=db._query(query).toArray();
NotethatanES6templatestringisusedforpopulatingthequeryvariable.ThestringisassembledusingtheaqlgeneratorfunctionwhichisbundledwithArangoDB.ThetemplatestringcancontainreferencestoJavaScriptvariablesorexpressionsvia${...}.Intheaboveexample,thequeryreferencesavariablenamedvalue.Theaqlfunctiongeneratesanobjectwithtwoseparateattributes:thequerystring,containingreferencestobindparameters,andtheactualbindparametervalues.
Bindparameternamesareautomaticallygeneratedbytheaqlfunction:
varvalue="someinputvalue";
aql`FORdocINcollectionFILTERdoc.value==${value}RETURNdoc`;
{
"query":"FORdocINcollectionFILTERdoc.value==@value0RETURNdoc",
"bindVars":{
"value0":"someinputvalue"
}
}
Usingbindparametersindynamicqueries
Bindparametersarehelpful,soitmakessensetousethemforhandlingthedynamicvalues.Youcanevenusethemforqueriesthatitselfarehighlydynamic,forexamplewithconditionalFILTERandLIMITparts.Here'showtodothis:
/*note:thisexamplehasaslightissue...hangonreading*/
varquery="FORdocINcollection";
varparams={};
if(useFilter){
query+="FILTERdoc.value==@what";
params.what=req.params("searchValue");
}
if(useLimit){
/*notquiteright,seebelow*/
query+="LIMIT@offset,@count";
params.offset=req.params("offset");
params.count=req.params("count");
}
query+="RETURNdoc";
db._query(query,params).toArray();
Notethatinthisexamplewe'rebacktostringconcatenation,butwithouttheproblemofthequerybeingvulnerabletoarbitrarymodifications.
Inputvaluevalidationandsanitation
Stillyoushouldprefertobeparanoid,andtrytodetectinvalidinputvaluesasearlyaspossible,atleastbeforeexecutingaquerywiththem.Thisisbecausesomeinputparametersmayaffecttheruntimebehaviorofqueriesnegativelyor,whenmodified,mayleadtoqueriesthrowingruntimeerrorsinsteadofreturningvalidresults.Thisisn'tsomethinganattackershoulddeserve.
LIMITisagoodexampleforthis:ifusedwithasingleargument,theargumentshouldbenumeric.WhenLIMITisgivenastringvalue,executingthequerywillfail.Youmaywanttodetectthisearlyanddon'treturnanHTTP500(asthiswouldsignalattackersthattheyweresuccessfulbreakingyourapplication).
AnotherproblemwithLIMITisthathighLIMITvaluesarelikelymoreexpensivethanlowones,andyoumaywanttodisallowusingLIMITvaluesexceedingacertainthreshold.
Here'swhatyoucoulddoinsuchcases:
varquery="FORdocINcollectionLIMIT@countRETURNdoc";
AvoidingParameterInjection
26
/*somedefaultvalueforlimit*/
varparams={count:100};
if(useLimit){
varcount=req.params("count");
/*abortifvaluedoesnotlooklikeaninteger*/
if(!preg_match(/^d+$/,count)){
throw"invalidcountvalue!";
}
/*actuallyturnitintoaninteger*/
params.count=parseInt(count,10);//turnintonumericvalue
}
if(params.count<1||params.count>1000){
/*valueisoutsideofacceptedthresholds*/
throw"invalidcountvalue!";
}
db._query(query,params).toArray();
Thisisabitmorecomplex,butthat'sapriceyou'relikelywillingtopayforabitofextrasafety.Inrealityyoumaywanttouseaframeworkforvalidation(suchasjoiwhichcomesbundledwithArangoDB)insteadofwritingyourownchecksallovertheplace.
Bindparametertypes
TherearetwotypesofbindparametersinAQL:
bindparametersforvalues:thoseareprefixedwithasingle@inAQLqueries,andarespecifiedwithouttheprefixwhentheygettheirvalueassigned.ThesebindparameterscancontainanyvalidJSONvalue.
Examples:@what,@searchValue
bindparametersforcollections:theseareprefixedwith@@inAQLqueries,andarereplacedwiththenameofacollection.Whenthebindparametervalueisassigned,theparameteritselfmustbespecifiedwithasingle@prefix.Onlystringvaluesareallowedforthistypeofbindparameters.
Examples:@@collection
Thelattertypeofbindparameterisprobablynotusedasoften,anditshouldnotbeusedtogetherwithuserinput.OtherwiseusersmayfreelydetermineonwhichcollectionyourAQLquerieswilloperate(note:thismaybeavalidusecase,butnormallyitisextremelyundesired).
Authors:JanSteemann
Tags:#injection#aql#security
AvoidingParameterInjection
27
Writingmulti-lineAQLqueries
Problem
IwanttowriteanAQLquerythatspansmultiplelinesinmyJavaScriptsourcecode,butitdoesnotwork.Howtodothis?
Solution
AQLsupportsmulti-linequeries,andtheAQLeditorinArangoDB'swebinterfacesupportsthemtoo.
Whenissuedprogrammatically,multi-linequeriescanbeasourceoferrors,atleastinsomelanguages.Forexample,JavaScriptisnotoriouslybadathandlingmulti-line(JavaScript)statements,anduntilrecentlyithadnosupportformulti-linestrings.
InJavaScript,therearethreewaysofwritingamulti-lineAQLqueryinthesourcecode:
stringconcatenationES6templatestringsquerybuilder
Whichmethodworksbestdependsonafewfactors,butisoftenenoughasimplematterofpreference.Beforedecidingonany,pleasemakesuretoreadtherecipeforavoidingparameterinjectiontoo.
Stringconcatenation
WewantthequeryFORdocINcollectionFILTERdoc.value==@whatRETURNdoctobecomemorelegibleinthesourcecode.
SimplysplittingthequerystringintothreelineswillleaveuswithaparseerrorinJavaScript:
/*willnotwork*/
varquery="FORdocINcollection
FILTERdoc.value==@what
RETURNdoc";
Instead,wecoulddothis:
varquery="FORdocINcollection"+
"FILTERdoc.value==@what"+
"RETURNdoc";
ThisisperfectlyvalidJavaScript,butit'serror-prone.Peoplehavespentagesonfindingsubtlebugsintheirqueriesbecausetheymissedasinglewhitespacecharacteratthebeginningorstartofsomeline.
Pleasenotethatwhenassemblingqueriesviastringconcatenation,youshouldstillusebindparameters(asdoneabovewith@what)andnotinsertuserinputvaluesintothequerystringwithoutsanitation.
ES6templatestrings
ES6templatestringsareeasiertogetrightandalsolookmoreelegant.TheycanbeusedinsideArangoDBsinceversion2.5.butsomeotherplatformsdon'tsupportthemet.Forexample,theycan'tbeusedinIEandoldernode.jsversions.Sousethemifyourenvironmentsupportsthemandyourcodedoesnotneedtorunonanynon-ES6environments.
Here'sthequerystringdeclaredviaanES6templatestring(notethatthestringmustbeenclosedinbackticksnow):
varquery=`FORdocINcollection
FILTERdoc.value==@what
RETURNdoc`;
Thewhitespaceinthetemplatestring-variantismucheasiertogetrightthanwhendoingthestringconcatenation.
MultilineQueryStrings
28
Thereareafewthingstonoteregardingtemplatestrings:
ES6templatestringscanbeusedtoinjectJavaScriptvaluesintothestringdynamically.Substitutionsstartwiththecharactersequence${.CaremustbetakenifthissequenceitselfisusedinsidetheAQLquerystring(currentlythiswouldbeinvalidAQL,butthismaychangeinfutureArangoDBversions).Additionally,anyvaluesinjectedintothequerystringusingparametersubstitutionswillnotbeescapedcorrectlyautomatically,soagainspecialcaremustbetakenwhenusingthismethodtokeepqueriessafefromparameterinjection.
amulti-linetemplatestringwillactuallycontainnewlinecharacters.Thisisnotnecessarilythecasewhendoingstringconcatenation.Inthestringconcatenationexample,weusedthreelinesofsourcecodetocreateasingle-linequerystring.Wecouldhaveinsertednewlinesintothequerystringtheretoo,butwedidn't.Justtopointoutthatthetwovariantswillnotcreatebytewise-identicalquerystrings.
PleasenotethatwhenusingES6templatestringsforyourqueries,youshouldstillusebindparameters(asdoneabovewith@what)andnotinsertuserinputvaluesintothequerystringwithoutsanitation.
ThereisaconveniencefunctionaqlwhichcanbeusedtosafelyandeasilybuildanAQLquerywithsubstitutionsfromarbitraryJavaScriptvaluesandexpressions.Itcanbeinvokedlikethis:
constaql=require("@arangodb").aql;//notneededinarangosh
varwhat="someinputvalue";
varquery=aql`FORdocINcollection
FILTERdoc.value==${what}
RETURNdoc`;
Thetemplatestringvariantthatusesaqlisbothconvenientandsafe.Internally,itwillturnthesubstitutedvaluesintobindparameters.Thequerystringandthebindparametervalueswillbereturnedseparately,sotheresultofqueryabovewillbesomethinglike:
{
"query":"FORdocINcollectionFILTERdoc.value==@value0RETURNdoc",
"bindVars":{
"value0":"someinputvalue"
}
}
Querybuilder
ArangoDBcomesbundledwithaquerybuildernamedaqb.ThatquerybuildercanbeusedtoprogrammaticallyconstructAQLqueries,withouthavingtowritequerystringsatall.
Here'sanexampleofitsusage:
varqb=require("aqb");
varjobs=db._createStatement({
query:(
qb.for('job').in('_jobs')
.filter(
qb('pending').eq('job.status')
.and(qb.ref('@queue').eq('job.queue'))
.and(qb.ref('@now').gte('job.delayUntil'))
)
.sort('job.delayUntil','ASC')
.limit('@max')
.return('job')
),
bindVars:{
queue:queue._key,
now:Date.now(),
max:queue.maxWorkers-numBusy
}
}).execute().toArray();
MultilineQueryStrings
29
Ascanbeseen,aqbprovidesafluentAPIthatallowschainingfunctioncallsforcreatingtheindividualqueryoperations.Thishasafewadvantages:
flexibility:thereisnoquerystringinthesourcecode,sothecodecanbeformattedasdesiredwithouthavingtobotheraboutstringsvalidation:thequerycanbevalidatedsyntacticallybyaqbbeforebeingactuallyexecutedbytheserver.Testingofqueriesalsobecomeseasier.Additionally,someIDEsmayprovideauto-completiontosomeextendandthusaiddevelopmentsecurity:built-inseparationofqueryoperations(e.g.FOR,FILTER,SORT,LIMIT)anddynamicvalues(e.g.userinputvalues)
aqbcanbeusedinsideArangoDBandfromnode.jsandevenfromwithinbrowsers.
Authors:JanSteemann
Tags:#aql#aqb#es6
MultilineQueryStrings
30
MigratingGRAPH_*Functionsfrom2.8orearlierto3.0
Problem
Withthereleaseof3.0allGRAPHfunctionshavebeendroppedfromAQLinfavorofamorenativeintegrationofgraphfeaturesintothequerylanguage.Ihaveusedtheoldgraphfunctionsandwanttoupgradeto3.0.
Graphfunctionscoveredinthisrecipe:
GRAPH_COMMON_NEIGHBORSGRAPH_COMMON_PROPERTIESGRAPH_DISTANCE_TOGRAPH_EDGESGRAPH_NEIGHBORSGRAPH_TRAVERSALGRAPH_TRAVERSAL_TREEGRAPH_SHORTEST_PATHGRAPH_PATHSGRAPH_VERTICES
Solution1:QuickandDirty(notrecommended)
Whentousethissolution
Iamnotwillingtoinvestalotiftimeintotheupgradeprocessandiamwillingtosurrendersomeperformanceinfavoroflesseffort.Someconstellationsmaynotworkwiththissolutionduetothenatureofuser-definedfunctions.EspeciallycheckforAQLqueriesthatdobothmodificationsandGRAPH_*functions.
Registeringuser-definedfunctions
ThisstephastobeexecutedonceonArangoDBforeverydatabaseweareusing.
Weconnecttoarangodbwitharangoshtoissuethefollowingcommandstwo:
vargraphs=require("@arangodb/general-graph");
graphs._registerCompatibilityFunctions();
ThesehaveregisteredalloldGRAPH_*functionsasuser-definedfunctionsagain,withtheprefixarangodb::.
Modifytheapplicationcode
NextwehavetogothroughourapplicationcodeandreplaceallcallstoGRAPH_*byarangodb::GRAPH_*.Nowrunatestrunofourapplicationandcheckifitworked.Ifitworkedwearereadytogo.
ImportantInformation
Theuserdefinedfunctionswillcalltranslatedsubqueries(asdescribedinSolution2).Theoptimizerdoesnotknowanythingaboutthesesubqueriesbeforehandandcannotoptimizethewholeplan.Alsotheremightberead/writeconstellationsthatareforbiddeninuser-definedfunctions,thereforea"really"translatedquerymayworkwhiletheuser-definedfunctionworkaroundmayberejected.
Solution2:Translatingthequeries(recommended)
Whentousethissolution
Migratingnamedgraphfunctionsto3.0
31
Iamwillingtoinvestsometimeonmyqueriesinordertogetmaximumperformance,fullqueryoptimizationandabettercontrolofmyqueries.Noforcingintotheoldlayoutanymore.
Beforeyoustart
IfyouareusingvertexExampleswhicharenotonly_idstringsdonotskiptheGRAPH_VERTICESsection,becauseitwilldescribehowtotranslatethemtoAQL.AllgraphfunctionsusingavertexExampleareidenticaltoexecutingaGRAPH_VERTICESbeforeandusingit'sresultasstartpoint.ExamplewithNEIGHBORS:
FORresINGRAPH_NEIGHBORS(@graph,@myExample)RETURNres
Isidenticalto:
FORstartGRAPH_VERTICES(@graph,@myExample)
FORresINGRAPH_NEIGHBORS(@graph,start)RETURNres
AllnonGRAPH_VERTICESfunctionswillonlyexplainthetransformationforasingleinputdocument's_id.
Optionsusedeverywhere
OptionedgeCollectionRestriction
InordertouseedgeCollectionrestrictionwejustusethefeaturethatthetraversercanwalkoveralistofedgecollectionsdirectly.SotheedgeCollectionRestrictionsjustformthislist(exampleGraphEdges):
//OLD
[..]FOReINGRAPH_EDGES(@graphName,@startId,{edgeCollectionRestriction:[edges1,edges2]})RETURNe
//NEW
[..]FORv,eINANY@startIdedges1,edges2RETURNDISTINCTe._id
Note:The@graphNamebindParameterisnotusedanymoreandprobablyhastoberemovedfromthequery.
OptionincludeData
IfweusetheoptionincludeDatawesimplyreturntheobjectdirectlyinsteadofonlythe_id
ExampleGRAPH_EDGES:
//OLD
[..]FOReINGRAPH_EDGES(@graphName,@startId,{includeData:true})RETURNe
//NEW
[..]FORv,eINANY@startIdGRAPH@graphNameRETURNDISTINCTe
Optiondirection
Thedirectionhastobeplacedbeforethestartid.Notehere:ThedirectionhastobeplacedasWorditcannotbehandedinviaabindParameteranymore:
//OLD
[..]FOReINGRAPH_EDGES(@graphName,@startId,{direction:'inbound'})RETURNe
//NEW
[..]FORv,eININBOUND@startIdGRAPH@graphNameRETURNDISTINCTe._id
OptionsminDepth,maxDepth
IfweusetheoptionsminDepthandmaxDepth(bothdefault1ifnotset)wecansimplyputtheminfrontofthedirectionpartintheTraversalstatement.
Migratingnamedgraphfunctionsto3.0
32
ExampleGRAPH_EDGES:
//OLD
[..]FOReINGRAPH_EDGES(@graphName,@startId,{minDepth:2,maxDepth:4})RETURNe
//NEW
[..]FORv,eIN2..4ANY@startIdGRAPH@graphNameRETURNDISTINCTe._id
OptionmaxIteration
TheoptionmaxIterationsisremovedwithoutreplacement.Yourqueriesarenowboundbymainmemorynotbyanarbitrarynumberofiterations.
GRAPH_VERTICES
Firstwehavetobranchontheexample.Therewehavethreepossibilities:
1. Theexampleisan_idstring.2. Theexampleisnullor{}.3. Theexampleisanonemptyobjectoranarray.
Exampleis'_id'string
Thisistheeasiestreplacement.InthiscasewesimplyreplacethefunctionwithacalltoDOCUMENT:
//OLD
[..]GRAPH_VERTICES(@graphName,@idString)[..]
//NEW
[..]DOCUMENT(@idString)[..]
NOTE:The@graphNameisnotrequiredanymore,wemayhavetoadjustbindParameters.
TheAQLgraphfeaturescanworkwithaniddirectly,noneedtocallDOCUMENTbeforeifwejustneedthistofindastartingpoint.
Exampleisnullortheemptyobject
Thiscasemeansweusealldocumentsfromthegraph.Herewefirsthavetonowthevertexcollectionsofthegraph.
1. Ifweonlyhaveonecollection(sayvertices)wecanreplaceitwithasimpleiterationoverthiscollection:
//OLD
[..]FORvINGRAPH_VERTICES(@graphName,{})[..]
//NEW
[..]FORvINvertices[..]
`
NOTE:The@graphNameisnotrequiredanymore,wemayhavetoadjustbindParameters.
1. Wehavemorethanonecollection.Thisistheunfortunatecaseforageneralreplacement.Sointhegeneralreplacementweassumewedonotwanttoexcludeanyofthecollectionsinthegraph.ThanweunfortunatelyhavetoformaUNIONoverallthesecollections.Sayourgraphhasthevertexcollectionsvertices1,vertices2,vertices3wecreateasub-queryforasinglecollectionforeachofthemandwraptheminacalltoUNION.
//OLD
[..]FORvINGRAPH_VERTICES(@graphName,{})[..]
//NEW
[..]
FORvINUNION(//WestartaUNION
(FORvINvertices1RETURNv),//Foreachvertexcollection
(FORvINvertices2RETURNv),//wecreatethesamesubquery
(FORvINvertices3RETURNv)
Migratingnamedgraphfunctionsto3.0
33
)//FinishwiththeUNION
[..]
`
NOTE:IfyouhaveanymoredomainknowledgeofyourgraphapplyitatthispointtoidentifywhichcollectionsareactuallyrelevantasthisUNIONisaratherexpensiveoperation.
IfweusetheoptionvertexCollectionRestrictionintheoriginalquery.TheUNIONhastobeformedbythecollectionsinthisrestrictioninsteadofALLcollections.
Exampleisanon-emptyobject
Firstwefollowtheinstructionsfortheemptyobjectabove.Inthissectionwewilljustfocusonasinglecollectionvertices,theUNIONformultiplecollectionsisagainwrappedaroundasubqueryforeachofthesecollectionsbuiltinthefollowingway.
NowwehavetotransformtheexampleintoanAQLFILTERstatement.Thereforewetakealltop-levelattributesoftheexampleanddoanequalcomparisonwiththeirvalues.AllofthesecomparisonsarejoinedwithanANDbecausetheallhavetobefulfilled.
Example:
//OLD
[..]FORvINGRAPH_VERTICES(@graphName,{foo:'bar',the:{answer:42}}})[..]
//NEW
[..]FORvINvertices
FILTERv.foo=='bar'//foo:bar
ANDv.the=={answer:42}//the:{answer:42}
[..]
Exampleisanarray
Theideatransformationisalmostidenticaltoasinglenon-emptyobject.ForeachelementinthearraywecreatethefilterconditionsandthanweOR-combinethem(mindthebrackets):
//OLD
[..]FORvINGRAPH_VERTICES(@graphName,[{foo:'bar',the:{answer:42}},{foo:'baz'}]))[..]
//NEW
[..]FORvINvertices
FILTER(v.foo=='bar'//foo:bar
ANDv.the=={answer:42})//the:{answer:42}
OR(v.foo=='baz')
[..]
GRAPH_EDGES
TheGRAPH_EDGEScanbesimplyreplacedbyacalltotheAQLtraversal.
Nooptions
ThedefaultoptionsdiduseadirectionANYandreturnedadistinctresultoftheedges.Alsoitdidjustreturntheedges_idvalue.
//OLD
[..]FOReINGRAPH_EDGES(@graphName,@startId)RETURNe
//NEW
[..]FORv,eINANY@startIdGRAPH@graphNameRETURNDISTINCTe._id
OptionedgeExamples.
SeeGRAPH_VERTICESonhowtotransformexamplestoAQLFILTER.Applythefilterontheedgevariablee.
GRAPH_NEIGHBORS
Migratingnamedgraphfunctionsto3.0
34
TheGRAPH_NEIGHBORSisabreadth-first-searchonthegraphwithaglobaluniquecheckforvertices.SowecanreplaceitbyaanAQLtraversalwiththeseoptions.
Nooptions
ThedefaultoptionsdiduseadirectionANYandreturnedadistinctresultoftheneighbors.Alsoitdidjustreturntheneighbors_idvalue.
//OLD
[..]FORnINGRAPH_NEIGHBORS(@graphName,@startId)RETURNn
//NEW
[..]FORnINANY@startIdGRAPH@graphNameOPTIONS{bfs:true,uniqueVertices:'global'}RETURNn
OptionneighborExamples
SeeGRAPH_VERTICESonhowtotransformexamplestoAQLFILTER.Applythefilterontheneighborvariablen.
OptionedgeExamples
SeeGRAPH_VERTICESonhowtotransformexamplestoAQLFILTER.Applythefilterontheedgevariablee.
Howeverthisisabitmorecomplicatedasitinterfereswiththeglobaluniquenesscheck.ForedgeExamplesitissufficentwhenanyedgepointingtotheneighbormatchesthefilter.Using{uniqueVertices:'global'}firstpicksanyedgerandomly.Thanitchecksagainstthisedgeonly.Ifweknowtherearenovertexpairswithmultipleedgesbetweenthemwecanusethesimplevariantwhichissave:
//OLD
[..]FORnINGRAPH_NEIGHBORS(@graphName,@startId,{edgeExample:{label:'friend'}})RETURNe
//NEW
[..]FORn,eINANY@startIdGRAPH@graphNameOPTIONS{bfs:true,uniqueVertices:'global'}FILTERe.label=='friend'RETURN
n._id
Iftheremaybemultipleedgesbetweenthesamepairofverticeswehavetomakethedistinctcheckourselfesandcannotrelyonthetraverserdoingitcorrectlyforus:
//OLD
[..]FORnINGRAPH_NEIGHBORS(@graphName,@startId,{edgeExample:{label:'friend'}})RETURNe
//NEW
[..]FORn,eINANY@startIdGRAPH@graphNameOPTIONS{bfs:true}FILTERe.label=='friend'RETURNDISTINCTn._id
OptionvertexCollectionRestriction
IfweusethevertexCollectionRestrictionwehavetopostFiltertheneighborsbasedontheircollection.ThereforewecanmakeuseofthefunctionIS_SAME_COLLECTION:
//OLD
[..]FORnINGRAPH_NEIGHBORS(@graphName,@startId,{vertexCollectionRestriction:['vertices1','vertices2']})RETURNe
//NEW
[..]FORnINANY@startIdGRAPH@graphNameOPTIONS{bfs:true,uniqueVertices:true}FILTERIS_SAME_COLLECTION(vertices1,n)O
RIS_SAME_COLLECTION(vertices2,n)RETURNDISTINCTn._id
GRAPH_COMMON_NEIGHBORS
GRAPH_COMMON_NEIGHBORSisdefinedastwoGRAPH_NEIGHBORSqueriesandthanformingtheINTERSECTIONofbothqueries.HowtotranslatetheoptionspleaserefertoGRAPH_NEIGHBORS.Finallywehavetobuildtheoldresultformat{left,right,neighbors}.Ifyoujustneedpartsoftheresultyoucanadaptthisquerytoyourspecificneeds.
//OLD
Migratingnamedgraphfunctionsto3.0
35
FORvINGRAPH_COMMON_NEIGHBORS(@graphName,'vertices/1','vertices/2',{direction:'any'})RETURNv
//NEW
LETn1=(//Neighborsforvertex1Example
FORnINANY'vertices/1'GRAPH'graph'OPTIONS{bfs:true,uniqueVertices:"global"}RETURNn._id
)
LETn2=(//Neighborsforvertex2Example
FORnINANY'vertices/2'GRAPH'graph'OPTIONS{bfs:true,uniqueVertices:"global"}RETURNn._id
)
LETcommon=INTERSECTION(n1,n2)//Gettheintersection
RETURN{//Producetheoriginalresult
left:'vertices/1',
right:'vertices/2,
neighbors:common
}
NOTE:Ifyouareusingexamplesinsteadof_idsyouhavetoaddafiltertomakesurethattheleftisnotequaltotherightstartvertex.Togiveyouanexamplewithasinglevertexcollectionvertices,thereplacementwouldlooklikethis:
//OLD
FORvINGRAPH_COMMON_NEIGHBORS(@graphName,{name:"Alice"},{name:"Bob"})RETURNv
//NEW
FORleftINvertices
FILTERleft.name=="Alice"
LETn1=(FORnINANYleftGRAPH'graph'OPTIONS{bfs:true,uniqueVertices:"global"}RETURNn._id)
FORrightINvertices
FILTERright.name=="Bob"
FILTERright!=left//Makesureleftisnotidenticaltoright
LETn2=(FORnINANYrightGRAPH'graph'OPTIONS{bfs:true,uniqueVertices:"global"}RETURNn._id)
LETneighbors=INTERSECTION(n1,n2)
FILTERLENGTH(neighbors)>0//Onlypairswithsharedneighborsshouldbereturned
RETURN{left:left._id,right:right._id,neighbors:neighbors}
GRAPH_PATHS
Thisfunctioncomputesallpathsoftheentiregraph(withagivenminDepthandmaxDepth)asyoucanimaginethisfeatureisextremelyexpensiveandshouldneverbeused.HoweverpathscanagainbereplacedbyAQLtraversal.Assumeweonlyhaveonevertexcollectionverticesagain.
Nooptions
Bydefaultpathsoflength0to10arereturned.Andcirclesarenotfollowed.
//OLD
RETURNGRAPH_PATHS('graph')
//NEW
FORstartINvertices
FORv,e,pIN0..10OUTBOUNDstartGRAPH'graph'RETURN{source:start,destination:v,edges:p.edges,vertices:p.vertices}
followCycles
IfthisoptionissetwehavetomodifytheoptionsofthetraversalbymodifyingtheuniqueEdgesproperty:
//OLD
RETURNGRAPH_PATHS('graph',{followCycles:true})
//NEW
FORstartINvertices
FORv,e,pIN0..10OUTBOUNDstartGRAPH'graph'OPTIONS{uniqueEdges:'none'}RETURN{source:start,destination:v,edges:p
.edges,vertices:p.vertices}
GRAPH_COMMON_PROPERTIES
Migratingnamedgraphfunctionsto3.0
36
Thisfeatureinvolvesseveralfull-collectionscansandthereforeisextremelyexpensive.IfyoureallyneedityoucantransformitwiththehelpofATTRIBUTES,KEEPandZIP.
Startwithsingle_id
//OLD
RETURNGRAPH_COMMON_PROPERTIES('graph',"vertices/1","vertices/2")
//NEW
LETleft=DOCUMENT("vertices/1")//getonedocument
LETright=DOCUMENT("vertices/2")//gettheotherone
LETshared=(FORaINATTRIBUTES(left)//findallsharedattributes
FILTERleft[a]==right[a]
ORa=='_id'//alwaysinclude_id
RETURNa)
FILTERLENGTH(shared)>1//Returnthemonlyiftheyshareanattribute
RETURNZIP([left._id],[KEEP(right,shared)])//Buildtheresult
StartwithvertexExamples
Againweassumeweonlyhaveasinglecollectionvertices.Wehavetotransformtheexamplesintofilters.Iterateoververticestofindallleftdocuments.Foreachleftdocumentiterateoververticesagaintofindmatchingrightdocuments.Andreturnthesharedattributesasabove:
//OLD
RETURNGRAPH_COMMON_PROPERTIES('graph',{answer:42},{foo:"bar"})
//NEW
FORleftINvertices
FILTERleft.answer==42
LETcommons=(
FORrightINvertices
FILTERright.foo=="bar"
FILTERleft!=right
LETshared=(FORaINATTRIBUTES(left)
FILTERleft[a]==right[a]
ORa=='_id'
RETURNa)
FILTERLENGTH(shared)>1
RETURNKEEP(right,shared))
FILTERLENGTH(commons)>0
RETURNZIP([left._id],[commons])
GRAPH_SHORTEST_PATH
AshortestpathcomputationisnowdoneviathenewSHORTEST_PATHAQLstatement.
Nooptions
//OLD
FORpINGRAPH_SHORTEST_PATH(@graphName,@startId,@targetId,{direction:'outbound'})RETURNp
//NEW
LETp=(//RunoneshortestPath
FORv,eINOUTBOUNDSHORTEST_PATH@startIdTO@targetIdGRAPH@graphName
//Wereturnobjectswithvertex,edgeandweightforeachvertexonthepath
RETURN{vertex:v,edge:e,weight:(IS_NULL(e)?0:1)}
)
FILTERLENGTH(p)>0//Weonlywantshortestpathsthatactuallyexist
RETURN{//Werebuildtheoldformat
vertices:p[*].vertex,
edges:p[*FILTERCURRENT.e!=null].edge,
distance:SUM(p[*].weight)
}
OptionsweightanddefaultWeight
Migratingnamedgraphfunctionsto3.0
37
ThenewAQLSHORTEST_PATHofferstheoptionsweightAttributeanddefaultWeight.
//OLD
FORpINGRAPH_SHORTEST_PATH(@graphName,@startId,@targetId,{direction:'outbound',weight:"weight",defaultWeight:80})RE
TURNp
//NEW
LETp=(//RunoneshortestPath
FORv,eINOUTBOUNDSHORTEST_PATH@startIdTO@targetIdGRAPH@graphName
//Wereturnobjectswithvertex,edgeandweightforeachvertexonthepath
RETURN{vertex:v,edge:e,weight:(IS_NULL(e)?0:(IS_NUMBER(e.weight)?e.weight:80))}
)
FILTERLENGTH(p)>0//Weonlywantshortestpathsthatactuallyexist
RETURN{//Werebuildtheoldformat
vertices:p[*].vertex,
edges:p[*FILTERCURRENT.e!=null].edge,
distance:SUM(p[*].weight)//Wehavetorecomputethedistanceifweneedit
}
GRAPH_DISTANCE_TO
GraphdistancetoonlydiffersbytheresultformatfromGRAPH_SHORTEST_PATH.SowefollowthetransformationforGRAPH_SHORTEST_PATH,removesomeunnecessaryparts,andchangethereturnformat
//OLD
FORpINGRAPH_DISTANCE_TO(@graphName,@startId,@targetId,{direction:'outbound'})RETURNp
//NEW
LETp=(//RunoneshortestPath
FORv,eINOUTBOUNDSHORTEST_PATH@startIdTO@targetIdGRAPH@graphName
//DIFFERENCEweonlyreturntheweightforeachedgeonthepath
RETURNIS_NULL(e)?0:1}
)
FILTERLENGTH(p)>0//Weonlywantshortestpathsthatactuallyexist
RETURN{//Werebuildtheoldformat
startVertex:@startId,
vertex:@targetId,
distance:SUM(p[*].weight)
}
GRAPH_TRAVERSALandGRAPH_TRAVERSAL_TREE
ThesehavebeenremovedandshouldbereplacedbythenativeAQLtraversal.Therearemanypotentialsolutionsusingthenewsyntax,buttheylargelydependonwhatexactlyyouaretryingtoachieveandwouldgobeyondthescopeofthiscookbook.Hereisoneexamplehowtodothetransition,usingtheworldgraphasdata:
In2.8,itwaspossibletouseGRAPH_TRAVERSAL()togetherwithacustomvisitorfunctiontofindleafnodesinagraph.Leafnodesareverticesthathaveinboundedges,butnooutboundedges.Thevisitorfunctioncodelookedlikethis:
varaqlfunctions=require("org/arangodb/aql/functions");
aqlfunctions.register("myfunctions::leafNodeVisitor",function(config,result,vertex,path,connected){
if(connected&&connected.length===0){
returnvertex.name+"("+vertex.type+")";
}
});
AndtheAQLquerytomakeuseofit:
LETparams={
order:"preorder-expander",
visitor:"myfunctions::leafNodeVisitor",
visitorReturnsResults:true
}
FORresultINGRAPH_TRAVERSAL("worldCountry","worldVertices/world","inbound",params)
RETURNresult
Migratingnamedgraphfunctionsto3.0
38
TotraversethegraphstartingatvertexworldVertices/worldusingnativeAQLtraversalandanamedgraph,wecansimplydo:
FORvIN0..10INBOUND"worldVertices/world"GRAPH"worldCountry"
RETURNv
Itwillgiveusallvertexdocumentsincludingthestartvertex(becausetheminimumdepthissetto0).Themaximumdepthissetto10,whichisenoughtofollowalledgesandreachtheleafnodesinthisgraph.
Thequerycanbemodifiedtoreturnaformattedpathfromfirsttolastnode:
FORv,e,pIN0..10INBOUND"worldVertices/world"GRAPH"worldCountry"
RETURNCONCAT_SEPARATOR("->",p.vertices[*].name)
Theresultlookslikethis(shortened):
[
"World",
"World->Africa",
"World->Africa->Coted'Ivoire",
"World->Africa->Coted'Ivoire->Yamoussoukro",
"World->Africa->Angola",
"World->Africa->Angola->Luanda",
"World->Africa->Chad",
"World->Africa->Chad->N'Djamena",
...
]
Aswecansee,allpossiblepathsofvaryinglengthsarereturned.Wearenotreallyinterestedinthem,butwestillhavetodothetraversaltogofromWorldallthewaytotheleafnodes(e.g.Yamoussoukro).Todetermineifavertexisreallythelastonthepathinthesenseofbeingaleafnode,wecanuseanothertraversalofdepth1tocheckifthereisatleastoneoutgoingedge-whichmeansthevertexisnotaleafnode,otherwiseitis:
FORvIN0..10INBOUND"worldVertices/world"GRAPH"worldCountry"
FILTERLENGTH(FORvvININBOUNDvGRAPH"worldCountry"LIMIT1RETURN1)==0
RETURNCONCAT(v.name,"(",v.type,")")
Usingthecurrentvertexvasstartingpoint,thesecondtraversalisperformed.Itcanreturnearlyafteroneedgewasfollowed(LIMIT1),becausewedon'tneedtoknowtheexactcountanditisfasterthisway.Wealsodon'tneedtheactualvertex,sowecanjustRETURN1asdummyvalueasanoptimization.Thetraversal(whichisasub-query)willreturnanemptyarrayincaseofaleafnode,and[1]otherwise.Sinceweonlywanttheleafnodes,weFILTERoutallnon-emptyarraysandwhatisleftaretheleafnodesonly.TheattributesnameandtypeareformattedthewaytheywerelikeintheoriginalJavaScriptcode,butnowwithAQL.Thefinalresultisalistofallcapitals:
[
"Yamoussoukro(capital)",
"Luanda(capital)",
"N'Djamena(capital)",
"Algiers(capital)",
"Yaounde(capital)",
"Ouagadougou(capital)",
"Gaborone(capital)",
"Asmara(capital)",
"Cairo(capital)",
...
]
ThereisnodirectsubstitutefortheGRAPH_TRAVERSAL_TREE()function.Theadvantageofthisfunctionwasthatits(possiblyhighlynested)resultdatastructureinherentlyrepresentedthe"longest"possiblepathsonly.WithnativeAQLtraversal,allpathsfromminimumtomaximumtraversaldeptharereturned,includingthe"short"pathsaswell:
FORv,e,pIN1..2INBOUND"worldVertices/continent-north-america"GRAPH"worldCountry"
RETURNCONCAT_SEPARATOR("<-",p.vertices[*]._key)
Migratingnamedgraphfunctionsto3.0
39
[
"continent-north-america<-country-antigua-and-barbuda",
"continent-north-america<-country-antigua-and-barbuda<-capital-saint-john-s",
"continent-north-america<-country-barbados",
"continent-north-america<-country-barbados<-capital-bridgetown",
"continent-north-america<-country-canada",
"continent-north-america<-country-canada<-capital-ottawa",
"continent-north-america<-country-bahamas",
"continent-north-america<-country-bahamas<-capital-nassau"
]
Asecondtraversalwithdepth=1canbeusedtocheckifwereachedaleafnode(nomoreincomingedges).Basedonthisinformation,the"short"pathscanbefilteredout.Notethatasecondconditionisrequired:itispossiblethatthelastnodeinatraversalisnotaleafnodeifthemaximumtraversaldepthisexceeded.Thus,weneedtoalsoletpathsthrough,whichcontainasmanyedgesashopswedointhetraversal(here:2).
FORv,e,pIN1..2INBOUND"worldVertices/continent-north-america"GRAPH"worldCountry"
LETother=(
FORvv,eeININBOUNDvGRAPH"worldCountry"
//FILTERee!=e//needediftraversingedgesinANYdirection
LIMIT1
RETURN1
)
FILTERLENGTH(other)==0||LENGTH(p.edges)==2
RETURNCONCAT_SEPARATOR("<-",p.vertices[*]._key)
[
"continent-north-america<-country-antigua-and-barbuda<-capital-saint-john-s",
"continent-north-america<-country-barbados<-capital-bridgetown",
"continent-north-america<-country-canada<-capital-ottawa",
"continent-north-america<-country-bahamas<-capital-nassau"
]
Thefullpathscanbereturned,butitisnotinatree-likestructureaswithGRAPH_TRAVERSAL_TREE().Suchadatastructurecanbeconstructedonclient-sideifreallyneeded.
FORv,e,pIN1..2INBOUND"worldVertices/continent-north-america"GRAPH"worldCountry"
LETother=(FORvv,eeININBOUNDvGRAPH"worldCountry"LIMIT1RETURN1)
FILTERLENGTH(other)==0||LENGTH(p.edges)==2
RETURNp
Pathdata(shortened):
[
{
"edges":[
{
"_id":"worldEdges/57585025",
"_from":"worldVertices/country-antigua-and-barbuda",
"_to":"worldVertices/continent-north-america",
"type":"is-in"
},
{
"_id":"worldEdges/57585231",
"_from":"worldVertices/capital-saint-john-s",
"_to":"worldVertices/country-antigua-and-barbuda",
"type":"is-in"
}
],
"vertices":[
{
"_id":"worldVertices/continent-north-america",
"name":"NorthAmerica",
"type":"continent"
},
{
"_id":"worldVertices/country-antigua-and-barbuda",
"code":"ATG",
Migratingnamedgraphfunctionsto3.0
40
"name":"AntiguaandBarbuda",
"type":"country"
},
{
"_id":"worldVertices/capital-saint-john-s",
"name":"SaintJohn's",
"type":"capital"
}
]
},
{
...
}
]
Thefirstandsecondvertexofthenthpathareconnectedbythefirstedge(p[n].vertices[0] p[n].edges[0]→p[n].vertices[1])andsoon.Thisstructuremightactuallybemoreconvenienttoprocesscomparedtoatree-likestructure.Notethattheedgedocumentsarealsoincluded,inconstrasttotheremovedgraphtraversalfunction.
Contactusviaoursocialchannelsifyouneedfurtherhelp.
Author:MichaelHackstein
Tags:#howto#aql#migration
Migratingnamedgraphfunctionsto3.0
41
MigratinganonymousgraphFunctionsfrom2.8orearlierto3.0
Problem
Withthereleaseof3.0allGRAPHfunctionshavebeendroppedfromAQLinfavorofamorenativeintegrationofgraphfeaturesintothequerylanguage.Ihaveusedtheoldgraphfunctionsandwanttoupgradeto3.0.
Graphfunctionscoveredinthisrecipe:
EDGESNEIGHBORSPATHSTRAVERSALTRAVERSAL_TREE
Solution
EDGES
TheEDGEScanbesimplyreplacedbyacalltotheAQLtraversal.
Nooptions
Thesyntaxisslightlydifferentbutmappingshouldbesimple:
//OLD
[..]FOReINEDGES(@@edgeCollection,@startId,'outbound')RETURNe
//NEW
[..]FORv,eINOUTBOUND@startId@@edgeCollectionRETURNe
UsingEdgeExamples
ExampleshavetobetransformedintoAQLfilterstatements.HowtodothispleasereadtheGRAPH_VERTICESsectioninMigratingGRAPH_*Functionsfrom2.8orearlierto3.0.Applythesefiltersontheedgevariablee.
OptionincluceVertices
Inordertoincludetheverticesyoujustusethevertexvariablevaswell:
//OLD
[..]FOReINEDGES(@@edgeCollection,@startId,'outbound',null,{includeVertices:true})RETURNe
//NEW
[..]FORv,eINOUTBOUND@startId@@edgeCollectionRETURN{edge:e,vertex:v}
NOTE:ThedirectioncannotbegivenasabindParameteranymoreithastobehard-codedinthequery.
NEIGHBORS
TheNEIGHBORSisabreadth-first-searchonthegraphwithaglobaluniquecheckforvertices.SowecanreplaceitbyaanAQLtraversalwiththeseoptions.Duetosyntaxchangesthevertexcollectionofthestartvertexisnolongermandatorytobegiven.YoumayhavetoadjustbindParameteresforthisquery.
Nooptions
Migratinganonymousgraphfunctionsto3.0
42
Thedefaultoptionsdidjustreturntheneighbors_idvalue.
//OLD
[..]FORnINNEIGHBORS(@@vertexCollection,@@edgeCollection,@startId,'outbound')RETURNn
//NEW
[..]FORnINOUTBOUND@startId@@edgeCollectionOPTIONS{bfs:true,uniqueVertices:'global'}RETURNn._id
NOTE:ThedirectioncannotbegivenasabindParameteranymoreithastobehard-codedinthequery.
UsingedgeExamples
ExampleshavetobetransformedintoAQLfilterstatements.HowtodothispleasereadtheGRAPH_VERTICESsectioninMigratingGRAPH_*Functionsfrom2.8orearlierto3.0.Applythesefiltersontheedgevariableewhichisthesecondreturnvariableofthetraversalstatement.
Howeverthisisabitmorecomplicatedasitinterfereswiththeglobaluniquenesscheck.ForedgeExamplesitissufficentwhenanyedgepointingtotheneighbormatchesthefilter.Using{uniqueVertices:'global'}firstpicksanyedgerandomly.Thanitchecksagainstthisedgeonly.Ifweknowtherearenovertexpairswithmultipleedgesbetweenthemwecanusethesimplevariantwhichissave:
//OLD
[..]FORnINNEIGHBORS(@@vertexCollection,@@edgeCollection,@startId,'outbound',{label:'friend'})RETURNn
//NEW
[..]FORn,eINOUTBOUND@startId@@edgeCollectionOPTIONS{bfs:true,uniqueVertices:'global'}
FILTERe.label=='friend'
RETURNn._id
Iftheremaybemultipleedgesbetweenthesamepairofverticeswehavetomakethedistinctcheckourselfesandcannotrelyonthetraverserdoingitcorrectlyforus:
//OLD
[..]FORnINNEIGHBORS(@@vertexCollection,@@edgeCollection,@startId,'outbound',{label:'friend'})RETURNn
//NEW
[..]FORn,eINOUTBOUND@startId@@edgeCollectionOPTIONS{bfs:true}
FILTERe.label=='friend'
RETURNDISTINCTn._id
OptionincludeData
Ifyouwanttoincludethedatasimplyreturnthecompletedocumentinsteadofonlythe_idvalue.
//OLD
[..]FORnINNEIGHBORS(@@vertexCollection,@@edgeCollection,@startId,'outbound',null,{includeData:true})RETURNn
//NEW
[..]FORn,eINOUTBOUND@startId@@edgeCollectionOPTIONS{bfs:true,uniqueVertices:'global'}RETURNn
PATHS
Thisfunctioncomputesallpathsoftheentireedgecollection(withagivenminDepthandmaxDepth)asyoucanimaginethisfeatureisextremelyexpensiveandshouldneverbeused.HoweverpathscanagainbereplacedbyAQLtraversal.
Nooptions
Bydefaultpathsoflength0to10arereturned.Andcirclesarenotfollowed.
//OLD
RETURNPATHS(@@vertexCollection,@@edgeCollection,"outbound")
//NEW
FORstartIN@@vertexCollection
Migratinganonymousgraphfunctionsto3.0
43
FORv,e,pIN0..10OUTBOUNDstart@@edgeCollectionRETURN{source:start,destination:v,edges:p.edges,vertices:p.vertice
s}
followCycles
IfthisoptionissetwehavetomodifytheoptionsofthetraversalbymodifyingtheuniqueEdgesproperty:
//OLD
RETURNPATHS(@@vertexCollection,@@edgeCollection,"outbound",{followCycles:true})
//NEW
FORstartIN@@vertexCollection
FORv,e,pIN0..10OUTBOUNDstart@@edgeCollectionOPTIONS{uniqueEdges:'none'}RETURN{source:start,destination:v,edges
:p.edges,vertices:p.vertices}
minDepthandmaxDepth
Ifthisoptionissetwehavetogivetheseparametersdirectlybeforethedirection.
//OLD
RETURNPATHS(@@vertexCollection,@@edgeCollection,"outbound",{minDepth:2,maxDepth:5})
//NEW
FORstartIN@@vertexCollection
FORv,e,pIN2..5OUTBOUNDstart@@edgeCollection
RETURN{source:start,destination:v,edges:p.edges,vertices:p.vertices}
TRAVERSALandTRAVERSAL_TREE
ThesehavebeenremovedandshouldbereplacedbythenativeAQLtraversal.Therearemanypotentialsolutionsusingthenewsyntax,buttheylargelydependonwhatexactlyyouaretryingtoachieveandwouldgobeyondthescopeofthiscookbook.Hereisoneexamplehowtodothetransition,usingtheworldgraphasdata:
In2.8,itwaspossibletouseTRAVERSAL()togetherwithacustomvisitorfunctiontofindleafnodesinagraph.Leafnodesareverticesthathaveinboundedges,butnooutboundedges.Thevisitorfunctioncodelookedlikethis:
varaqlfunctions=require("org/arangodb/aql/functions");
aqlfunctions.register("myfunctions::leafNodeVisitor",function(config,result,vertex,path,connected){
if(connected&&connected.length===0){
returnvertex.name+"("+vertex.type+")";
}
});
AndtheAQLquerytomakeuseofit:
LETparams={
order:"preorder-expander",
visitor:"myfunctions::leafNodeVisitor",
visitorReturnsResults:true
}
FORresultINTRAVERSAL(worldVertices,worldEdges,"worldVertices/world","inbound",params)
RETURNresult
TotraversethegraphstartingatvertexworldVertices/worldusingnativeAQLtraversalandananonymousgraph,wecansimplydo:
FORvIN0..10INBOUND"worldVertices/world"worldEdges
RETURNv
Itwillgiveusallvertexdocumentsincludingthestartvertex(becausetheminimumdepthissetto0).Themaximumdepthissetto10,whichisenoughtofollowalledgesandreachtheleafnodesinthisgraph.
Thequerycanbemodifiedtoreturnaformattedpathfromfirsttolastnode:
Migratinganonymousgraphfunctionsto3.0
44
FORv,e,pIN0..10INBOUND"worldVertices/world"e
RETURNCONCAT_SEPARATOR("->",p.vertices[*].name)
Theresultlookslikethis(shortened):
[
"World",
"World->Africa",
"World->Africa->Coted'Ivoire",
"World->Africa->Coted'Ivoire->Yamoussoukro",
"World->Africa->Angola",
"World->Africa->Angola->Luanda",
"World->Africa->Chad",
"World->Africa->Chad->N'Djamena",
...
]
Aswecansee,allpossiblepathsofvaryinglengthsarereturned.Wearenotreallyinterestedinthem,butwestillhavetodothetraversaltogofromWorldallthewaytotheleafnodes(e.g.Yamoussoukro).Todetermineifavertexisreallythelastonthepathinthesenseofbeingaleafnode,wecanuseanothertraversalofdepth1tocheckifthereisatleastoneoutgoingedge-whichmeansthevertexisnotaleafnode,otherwiseitis:
FORvIN0..10INBOUND"worldVertices/world"worldEdges
FILTERLENGTH(FORvvININBOUNDvworldEdgesLIMIT1RETURN1)==0
RETURNCONCAT(v.name,"(",v.type,")")
Usingthecurrentvertexvasstartingpoint,thesecondtraversalisperformed.Itcanreturnearlyafteroneedgewasfollowed(LIMIT1),becausewedon'tneedtoknowtheexactcountanditisfasterthisway.Wealsodon'tneedtheactualvertex,sowecanjustRETURN1asdummyvalueasanoptimization.Thetraversal(whichisasub-query)willreturnanemptyarrayincaseofaleafnode,and[1]otherwise.Sinceweonlywanttheleafnodes,weFILTERoutallnon-emptyarraysandwhatisleftaretheleafnodesonly.TheattributesnameandtypeareformattedthewaytheywerelikeintheoriginalJavaScriptcode,butnowwithAQL.Thefinalresultisalistofallcapitals:
[
"Yamoussoukro(capital)",
"Luanda(capital)",
"N'Djamena(capital)",
"Algiers(capital)",
"Yaounde(capital)",
"Ouagadougou(capital)",
"Gaborone(capital)",
"Asmara(capital)",
"Cairo(capital)",
...
]
ThereisnodirectsubstitutefortheTRAVERSAL_TREE()function.Theadvantageofthisfunctionwasthatits(possiblyhighlynested)resultdatastructureinherentlyrepresentedthe"longest"possiblepathsonly.WithnativeAQLtraversal,allpathsfromminimumtomaximumtraversaldeptharereturned,includingthe"short"pathsaswell:
FORv,e,pIN1..2INBOUND"worldVertices/continent-north-america"worldEdges
RETURNCONCAT_SEPARATOR("<-",p.vertices[*]._key)
[
"continent-north-america<-country-antigua-and-barbuda",
"continent-north-america<-country-antigua-and-barbuda<-capital-saint-john-s",
"continent-north-america<-country-barbados",
"continent-north-america<-country-barbados<-capital-bridgetown",
"continent-north-america<-country-canada",
"continent-north-america<-country-canada<-capital-ottawa",
"continent-north-america<-country-bahamas",
"continent-north-america<-country-bahamas<-capital-nassau"
]
Migratinganonymousgraphfunctionsto3.0
45
Asecondtraversalwithdepth=1canbeusedtocheckifwereachedaleafnode(nomoreincomingedges).Basedonthisinformation,the"short"pathscanbefilteredout.Notethatasecondconditionisrequired:itispossiblethatthelastnodeinatraversalisnotaleafnodeifthemaximumtraversaldepthisexceeded.Thus,weneedtoalsoletpathsthrough,whichcontainasmanyedgesashopswedointhetraversal(here:2).
FORv,e,pIN1..2INBOUND"worldVertices/continent-north-america"worldEdges
LETother=(
FORvv,eeININBOUNDvworldEdges
//FILTERee!=e//needediftraversingedgesinANYdirection
LIMIT1
RETURN1
)
FILTERLENGTH(other)==0||LENGTH(p.edges)==2
RETURNCONCAT_SEPARATOR("<-",p.vertices[*]._key)
[
"continent-north-america<-country-antigua-and-barbuda<-capital-saint-john-s",
"continent-north-america<-country-barbados<-capital-bridgetown",
"continent-north-america<-country-canada<-capital-ottawa",
"continent-north-america<-country-bahamas<-capital-nassau"
]
Thefullpathscanbereturned,butitisnotinatree-likestructureaswithTRAVERSAL_TREE().Suchadatastructurecanbeconstructedonclient-sideifreallyneeded.
FORv,e,pIN1..2INBOUND"worldVertices/continent-north-america"worldEdges
LETother=(FORvv,eeININBOUNDvworldEdgesLIMIT1RETURN1)
FILTERLENGTH(other)==0||LENGTH(p.edges)==2
RETURNp
Pathdata(shortened):
[
{
"edges":[
{
"_id":"worldEdges/57585025",
"_from":"worldVertices/country-antigua-and-barbuda",
"_to":"worldVertices/continent-north-america",
"type":"is-in"
},
{
"_id":"worldEdges/57585231",
"_from":"worldVertices/capital-saint-john-s",
"_to":"worldVertices/country-antigua-and-barbuda",
"type":"is-in"
}
],
"vertices":[
{
"_id":"worldVertices/continent-north-america",
"name":"NorthAmerica",
"type":"continent"
},
{
"_id":"worldVertices/country-antigua-and-barbuda",
"code":"ATG",
"name":"AntiguaandBarbuda",
"type":"country"
},
{
"_id":"worldVertices/capital-saint-john-s",
"name":"SaintJohn's",
"type":"capital"
}
]
},
{
...
Migratinganonymousgraphfunctionsto3.0
46
}
]
Thefirstandsecondvertexofthenthpathareconnectedbythefirstedge(p[n].vertices[0] p[n].edges[0]→p[n].vertices[1])andsoon.Thisstructuremightactuallybemoreconvenienttoprocesscomparedtoatree-likestructure.Notethattheedgedocumentsarealsoincluded,inconstrasttotheremovedgraphtraversalfunction.
Contactusviaoursocialchannelsifyouneedfurtherhelp.
Author:MichaelHackstein
Tags:#howto#aql#migration
Migratinganonymousgraphfunctionsto3.0
47
MigratingGRAPH_*Measurementsfrom2.8orearlierto3.0
Problem
Withthereleaseof3.0allGRAPHfunctionshavebeendroppedfromAQLinfavorofamorenativeintegrationofgraphfeaturesintothequerylanguage.Ihaveusedtheoldgraphfunctionsandwanttoupgradeto3.0.
Graphfunctionscoveredinthisrecipe:
GRAPH_ABSOLUTE_BETWEENNESSGRAPH_ABSOLUTE_CLOSENESSGRAPH_ABSOLUTE_ECCENTRICITYGRAPH_BETWEENNESSGRAPH_CLOSENESSGRAPH_DIAMETERGRAPH_ECCENTRICITYGRAPH_RADIUS
Solution1:UserDefinedFuntions
Registeringuser-definedfunctions
ThisstephastobeexecutedonceonArangoDBforeverydatabaseweareusing.
Weconnecttoarangodbwitharangoshtoissuethefollowingcommandstwo:
vargraphs=require("@arangodb/general-graph");
graphs._registerCompatibilityFunctions();
ThesehaveregisteredalloldGRAPH_*functionsasuser-definedfunctionsagain,withtheprefixarangodb::.
Modifytheapplicationcode
NextwehavetogothroughourapplicationcodeandreplaceallcallstoGRAPH_*byarangodb::GRAPH_*.Nowrunatestrunofourapplicationandcheckifitworked.Ifitworkedwearereadytogo.
ImportantInformation
Theuserdefinedfunctionswillcalltranslatedsubqueries(asdescribedinSolution2).Theoptimizerdoesnotknowanythingaboutthesesubqueriesbeforehandandcannotoptimizethewholeplan.Alsotheremightberead/writeconstellationsthatareforbiddeninuser-definedfunctions,thereforea"really"translatedquerymayworkwhiletheuser-definedfunctionworkaroundmayberejected.
Solution2:Foxx(recommended)Thegeneralgraphmodulestilloffersthemeasurementfunctions.AsthesearetypicallycomputationexpensiveandcreatelongrunningqueriesitisrecommendedtonotusethemincombinationwithotherAQLfeatures.ThereforethebestideaistoofferthesemeasurementsdirectlyviaanAPIusingFOXX.
FirstwecreateanewFoxxservice.Thenweincludethegeneral-graphmoduleintheservice.ForeverymeasurementweneedwesimplyofferaGETroutetoreadthismeasurement.
AsanexamplewedotheGRAPH_RADIUS:
///ADDFOXXCODEABOVE
constjoi=require('joi');
Migratinggraphmeasurementsto3.0
48
constcreateRouter=require('@arangodb/foxx/router');
constdd=require('dedent');
constrouter=createRouter();
constgraphs=require("@arangodb/general-graph");
router.get('/radius/:graph',function(req,res){
letgraph;
//Loadthegraph
try{
graph=graphs._graph(req.graph);
}catch(e){
res.throw('notfound');
}
res.json(graphs._radius());//Returntheradius
})
.pathParam('graph',joi.string().required(),'Thenameofthegraph')
.error('notfound','Graphwiththisnamedoesnotexist.')
.summary('ComputetheRadius')
.description(dd`
Thisfunctioncomputestheradiusofthegivengraph
andreturnsit.
`);
Author:MichaelHackstein
Tags:#howto#aql#migration
Migratinggraphmeasurementsto3.0
49
GraphFulldepthGraph-Traversal
UsingacustomVisitor
ExampleAQLQueriesforGraphs
Graph
50
FulldepthGraph-Traversal
Problem
Letsassumeyouhaveadatabaseandsomeedgesandvertices.Nowyouneedthenodewiththemostconnectionsinfulldepth.
Solution
Youneedacustomtraversalwiththefollowingproperties:
StoreallverticesyouhavevisitedalreadyIfyouvisitanalreadyvisitedvertexreturntheconnections+1anddonottouchtheedgesIfyouvisitafreshvertexvisitallitschildrenandsumuptheirconnections.Storethissumandreturnit+1Repeatforallvertices.
vartraversal=require("org/arangodb/graph/traversal");
varknownFilter=function(config,vertex,path){
if(config.known[vertex._key]!==undefined){
return"prune";
}
return"";
};
varsumVisitor=function(config,result,vertex,path){
if(config.known[vertex._key]!==undefined){
result.sum+=config.known[vertex._key];
}else{
config.known[vertex._key]=result.sum;
}
result.sum+=1;
return;
};
varconfig={
datasource:traversal.collectionDatasourceFactory(db.e),//eismyedgecollection
strategy:"depthfirst",
order:"preorder",
filter:knownFilter,
expander:traversal.outboundExpander,
visitor:sumVisitor,
known:{}
};
vartraverser=newtraversal.Traverser(config);
varcursor=db.v.all();//vismyvertexcollection
while(cursor.hasNext()){
varnode=cursor.next();
traverser.traverse({sum:0},node);
}
config.known;//Returnstheresultoftypename:counter.Inarangoshthiswillprintoutcompleteresult
Toexecutethisscriptaccordinglyreplacedb.vanddb.ewithyourcollections(visvertices,eisedges)andwriteittoafile:(e.g.)traverse.jsthenexecuteitinarangosh:
cattraverse.js|arangosh
IfyouwanttouseitinproductionyoushouldhavealookattheFoxxframeworkwhichallowsyoutostoreandexecutethisscriptonserversideandmakeitaccessibleviayourownAPI:Foxx
Comment
FulldepthGraph-Traversal
51
Youonlycomputetheconnectionsofonevertexonceandcacheitthen.Complexityisalmostequaltotheamountofedges.Inthecodebelowconfig.knowncontainstheresultofallvertices,youthencanaddthesortingonit.
Author:MichaelHackstein
Tags:#graph
FulldepthGraph-Traversal
52
Usingacustomvisitorfromnode.js
Problem
Iwanttotraverseagraphusingacustomvisitorfromnode.js.
Solution
UsearangojsandanAQLquerywithacustomvisitor.
Installingarangojs
Firstthingistoinstallarangojs.Thiscanbedoneusingnpmorbower:
npminstallarangojs
or
bowerinstallarangojs
Exampledatasetup
Forthefollowingexample,weneedtheexamplegraphanddatafromhere.Pleasedownloadthecodefromthelinkandstoreitinthefilesystemusingafilenameofworld-graph-setup.js.ThenstarttheArangoShellandrunthecodefromthefile:
require("internal").load("/path/to/file/world-graph-setup.js");
Thescriptwillcreatethefollowingtwocollectionsandloadsomedataintothem:
v:acollectionwithvertexdocumentse:anedgecollectioncontainingtheconnectionsbetweenverticesinv
Registeringacustomvisitorfunction
Let'sregisteracustomvisitorfunctionnow.AcustomvisitorfunctionisaJavaScriptfunctionthatisexecutedeverytimethetraversalprocessesavertexinthegraph.
Toregisteracustomvisitorfunction,wecanexecutethefollowingcommandsintheArangoShell:
varaqlfunctions=require("org/arangodb/aql/functions");
aqlfunctions.register("myfunctions::leafNodeVisitor",function(config,result,vertex,path,connected){
if(connected&&connected.length===0){
returnvertex.name+"("+vertex.type+")";
}
});
Invokingthecustomvisitor
Thefollowingcodecanberuninnode.jstoexecuteanAQLquerythatwillmakeuseofthecustomvisitor:
Database=require('arangojs');
/*connectionthedatabase,changeasrequired*/
db=newDatabase('http://127.0.0.1:8529');
/*thequerystring*/
UsingacustomVisitor
53
varquery="FORresultINTRAVERSAL(v,e,@vertex,'inbound',@options)RETURNresult";
/*bindparameters*/
varbindVars={
vertex:"v/world",/*ourstartvertex*/
options:{
order:"preorder-expander",
visitor:"myfunctions::leafNodeVisitor",
visitorReturnsResults:true
}
};
db.query(query,bindVars,function(err,cursor){
if(err){
console.log('error:%j',err);
}else{
cursor.all(function(err2,list){
if(err){
console.log('error:%j',err2);
}else{
console.log("alldocumentkeys:%j",list);
}
});
}
});
Author:JanSteemann
Tags:#graph#traversal#aql#nodejs
UsingacustomVisitor
54
AQLExampleQueriesonanActorsandMoviesDatabase
Acknowledgments
OnStackoverflowtheuserVinczaskedforsomeexamplequeriesbasedongraphs.Socreditsforthisquestionsgotohim.Thedatasetsandquerieshavebeentakenfromtheguysofneo4j.Creditsandthankstothem.AsIalsothinkthisexamplesareyetmissingIdecidedtowritethisrecipe.
Problem
(CopyfromStackoverflow)Givenacollectionofactorsandacollectionofmovies.AndaactInedgescollection(withayearproperty)toconnectthevertex.
[Actor]←actin→[Movie]
HowcouldIget:
Allactorswhoactedin"movie1"OR"movie2"Allactorswhoactedinboth"movie1"AND"movie2"?Allcommonmoviesbetween"actor1"and"actor2"?Allactorswhoactedin3ormoremovies?Allmovieswhereexactly6actorsactedin?Thenumberofactorsbymovie?Thenumberofmoviesbyactor?Thenumberofmoviesactedinbetween2005and2010byactor?
Solution
Duringthissolutionwewillbeusingarangoshtocreateandquerythedata.AlltheAQLqueriesarestringsandcansimplybecopiedovertoyourfavoritedriverinsteadofarangosh.
CreateaTestDatasetinarangosh:
varactors=db._create("actors");
varmovies=db._create("movies");
varactsIn=db._createEdgeCollection("actsIn");
varTheMatrix=movies.save({_key:"TheMatrix",title:'TheMatrix',released:1999,tagline:'WelcometotheRealWorld'})._id;
varKeanu=actors.save({_key:"Keanu",name:'KeanuReeves',born:1964})._id;
varCarrie=actors.save({_key:"Carrie",name:'Carrie-AnneMoss',born:1967})._id;
varLaurence=actors.save({_key:"Laurence",name:'LaurenceFishburne',born:1961})._id;
varHugo=actors.save({_key:"Hugo",name:'HugoWeaving',born:1960})._id;
varEmil=actors.save({_key:"Emil",name:"EmilEifrem",born:1978});
actsIn.save(Keanu,TheMatrix,{roles:["Neo"],year:1999});
actsIn.save(Carrie,TheMatrix,{roles:["Trinity"],year:1999});
actsIn.save(Laurence,TheMatrix,{roles:["Morpheus"],year:1999});
actsIn.save(Hugo,TheMatrix,{roles:["AgentSmith"],year:1999});
actsIn.save(Emil,TheMatrix,{roles:["Emil"],year:1999});
varTheMatrixReloaded=movies.save({_key:"TheMatrixReloaded",title:"TheMatrixReloaded",released:2003,tagline:"Freeyo
urmind"});
actsIn.save(Keanu,TheMatrixReloaded,{roles:["Neo"],year:2003});
actsIn.save(Carrie,TheMatrixReloaded,{roles:["Trinity"],year:2003});
actsIn.save(Laurence,TheMatrixReloaded,{roles:["Morpheus"],year:2003});
actsIn.save(Hugo,TheMatrixReloaded,{roles:["AgentSmith"],year:2003});
varTheMatrixRevolutions=movies.save({_key:"TheMatrixRevolutions",title:"TheMatrixRevolutions",released:2003,tagline:
"Everythingthathasabeginninghasanend"});
actsIn.save(Keanu,TheMatrixRevolutions,{roles:["Neo"],year:2003});
actsIn.save(Carrie,TheMatrixRevolutions,{roles:["Trinity"],year:2003});
actsIn.save(Laurence,TheMatrixRevolutions,{roles:["Morpheus"],year:2003});
ExampleAQLQueriesforGraphs
55
actsIn.save(Hugo,TheMatrixRevolutions,{roles:["AgentSmith"],year:2003});
varTheDevilsAdvocate=movies.save({_key:"TheDevilsAdvocate",title:"TheDevil'sAdvocate",released:1997,tagline:'Evilhas
itswinningways'})._id;
varCharlize=actors.save({_key:"Charlize",name:'CharlizeTheron',born:1975})._id;
varAl=actors.save({_key:"Al",name:'AlPacino',born:1940})._id;
actsIn.save(Keanu,TheDevilsAdvocate,{roles:["KevinLomax"],year:1997});
actsIn.save(Charlize,TheDevilsAdvocate,{roles:["MaryAnnLomax"],year:1997});
actsIn.save(Al,TheDevilsAdvocate,{roles:["JohnMilton"],year:1997});
varAFewGoodMen=movies.save({_key:"AFewGoodMen",title:"AFewGoodMen",released:1992,tagline:"Intheheartofthenation'
scapital,inacourthouseoftheU.S.government,onemanwillstopatnothingtokeephishonor,andonewillstopatnothing
tofindthetruth."})._id;
varTomC=actors.save({_key:"TomC",name:'TomCruise',born:1962})._id;
varJackN=actors.save({_key:"JackN",name:'JackNicholson',born:1937})._id;
varDemiM=actors.save({_key:"DemiM",name:'DemiMoore',born:1962})._id;
varKevinB=actors.save({_key:"KevinB",name:'KevinBacon',born:1958})._id;
varKieferS=actors.save({_key:"KieferS",name:'KieferSutherland',born:1966})._id;
varNoahW=actors.save({_key:"NoahW",name:'NoahWyle',born:1971})._id;
varCubaG=actors.save({_key:"CubaG",name:'CubaGoodingJr.',born:1968})._id;
varKevinP=actors.save({_key:"KevinP",name:'KevinPollak',born:1957})._id;
varJTW=actors.save({_key:"JTW",name:'J.T.Walsh',born:1943})._id;
varJamesM=actors.save({_key:"JamesM",name:'JamesMarshall',born:1967})._id;
varChristopherG=actors.save({_key:"ChristopherG",name:'ChristopherGuest',born:1948})._id;
actsIn.save(TomC,AFewGoodMen,{roles:['Lt.DanielKaffee'],year:1992});
actsIn.save(JackN,AFewGoodMen,{roles:['Col.NathanR.Jessup'],year:1992});
actsIn.save(DemiM,AFewGoodMen,{roles:['Lt.Cdr.JoAnneGalloway'],year:1992});
actsIn.save(KevinB,AFewGoodMen,{roles:['Capt.JackRoss'],year:1992});
actsIn.save(KieferS,AFewGoodMen,{roles:['Lt.JonathanKendrick'],year:1992});
actsIn.save(NoahW,AFewGoodMen,{roles:['Cpl.JeffreyBarnes'],year:1992});
actsIn.save(CubaG,AFewGoodMen,{roles:['Cpl.CarlHammaker'],year:1992});
actsIn.save(KevinP,AFewGoodMen,{roles:['Lt.SamWeinberg'],year:1992});
actsIn.save(JTW,AFewGoodMen,{roles:['Lt.Col.MatthewAndrewMarkinson'],year:1992});
actsIn.save(JamesM,AFewGoodMen,{roles:['Pfc.LoudenDowney'],year:1992});
actsIn.save(ChristopherG,AFewGoodMen,{roles:['Dr.Stone'],year:1992});
varTopGun=movies.save({_key:"TopGun",title:"TopGun",released:1986,tagline:'Ifeeltheneed,theneedforspeed.'})._id;
varKellyM=actors.save({_key:"KellyM",name:'KellyMcGillis',born:1957})._id;
varValK=actors.save({_key:"ValK",name:'ValKilmer',born:1959})._id;
varAnthonyE=actors.save({_key:"AnthonyE",name:'AnthonyEdwards',born:1962})._id;
varTomS=actors.save({_key:"TomS",name:'TomSkerritt',born:1933})._id;
varMegR=actors.save({_key:"MegR",name:'MegRyan',born:1961})._id;
actsIn.save(TomC,TopGun,{roles:['Maverick'],year:1986});
actsIn.save(KellyM,TopGun,{roles:['Charlie'],year:1986});
actsIn.save(ValK,TopGun,{roles:['Iceman'],year:1986});
actsIn.save(AnthonyE,TopGun,{roles:['Goose'],year:1986});
actsIn.save(TomS,TopGun,{roles:['Viper'],year:1986});
actsIn.save(MegR,TopGun,{roles:['Carole'],year:1986});
varJerryMaguire=movies.save({_key:"JerryMaguire",title:'JerryMaguire',released:2000,tagline:'Therestofhislifebegins
now.'})._id;
varReneeZ=actors.save({_key:"ReneeZ",name:'ReneeZellweger',born:1969})._id;
varKellyP=actors.save({_key:"KellyP",name:'KellyPreston',born:1962})._id;
varJerryO=actors.save({_key:"JerryO",name:"JerryO'Connell",born:1974})._id;
varJayM=actors.save({_key:"JayM",name:'JayMohr',born:1970})._id;
varBonnieH=actors.save({_key:"BonnieH",name:'BonnieHunt',born:1961})._id;
varReginaK=actors.save({_key:"ReginaK",name:'ReginaKing',born:1971})._id;
varJonathanL=actors.save({_key:"JonathanL",name:'JonathanLipnicki',born:1996})._id;
actsIn.save(TomC,JerryMaguire,{roles:['JerryMaguire'],year:2000});
actsIn.save(CubaG,JerryMaguire,{roles:['RodTidwell'],year:2000});
actsIn.save(ReneeZ,JerryMaguire,{roles:['DorothyBoyd'],year:2000});
actsIn.save(KellyP,JerryMaguire,{roles:['AveryBishop'],year:2000});
actsIn.save(JerryO,JerryMaguire,{roles:['FrankCushman'],year:2000});
actsIn.save(JayM,JerryMaguire,{roles:['BobSugar'],year:2000});
actsIn.save(BonnieH,JerryMaguire,{roles:['LaurelBoyd'],year:2000});
actsIn.save(ReginaK,JerryMaguire,{roles:['MarceeTidwell'],year:2000});
actsIn.save(JonathanL,JerryMaguire,{roles:['RayBoyd'],year:2000});
varStandByMe=movies.save({_key:"StandByMe",title:"StandByMe",released:1986,tagline:"Forsome,it'sthelastrealtaste
ofinnocence,andthefirstrealtasteoflife.Butforeveryone,it'sthetimethatmemoriesaremadeof."})._id;
varRiverP=actors.save({_key:"RiverP",name:'RiverPhoenix',born:1970})._id;
varCoreyF=actors.save({_key:"CoreyF",name:'CoreyFeldman',born:1971})._id;
varWilW=actors.save({_key:"WilW",name:'WilWheaton',born:1972})._id;
varJohnC=actors.save({_key:"JohnC",name:'JohnCusack',born:1966})._id;
varMarshallB=actors.save({_key:"MarshallB",name:'MarshallBell',born:1942})._id;
actsIn.save(WilW,StandByMe,{roles:['GordieLachance'],year:1986});
ExampleAQLQueriesforGraphs
56
actsIn.save(RiverP,StandByMe,{roles:['ChrisChambers'],year:1986});
actsIn.save(JerryO,StandByMe,{roles:['VernTessio'],year:1986});
actsIn.save(CoreyF,StandByMe,{roles:['TeddyDuchamp'],year:1986});
actsIn.save(JohnC,StandByMe,{roles:['DennyLachance'],year:1986});
actsIn.save(KieferS,StandByMe,{roles:['AceMerrill'],year:1986});
actsIn.save(MarshallB,StandByMe,{roles:['Mr.Lachance'],year:1986});
varAsGoodAsItGets=movies.save({_key:"AsGoodAsItGets",title:'AsGoodasItGets',released:1997,tagline:'Acomedyfromthe
heartthatgoesforthethroat.'})._id;
varHelenH=actors.save({_key:"HelenH",name:'HelenHunt',born:1963})._id;
varGregK=actors.save({_key:"GregK",name:'GregKinnear',born:1963})._id;
actsIn.save(JackN,AsGoodAsItGets,{roles:['MelvinUdall'],year:1997});
actsIn.save(HelenH,AsGoodAsItGets,{roles:['CarolConnelly'],year:1997});
actsIn.save(GregK,AsGoodAsItGets,{roles:['SimonBishop'],year:1997});
actsIn.save(CubaG,AsGoodAsItGets,{roles:['FrankSachs'],year:1997});
varWhatDreamsMayCome=movies.save({_key:"WhatDreamsMayCome",title:'WhatDreamsMayCome',released:1998,tagline:'Afterlife
thereismore.Theendisjustthebeginning.'})._id;
varAnnabellaS=actors.save({_key:"AnnabellaS",name:'AnnabellaSciorra',born:1960})._id;
varMaxS=actors.save({_key:"MaxS",name:'MaxvonSydow',born:1929})._id;
varWernerH=actors.save({_key:"WernerH",name:'WernerHerzog',born:1942})._id;
varRobin=actors.save({_key:"Robin",name:'RobinWilliams',born:1951})._id;
actsIn.save(Robin,WhatDreamsMayCome,{roles:['ChrisNielsen'],year:1998});
actsIn.save(CubaG,WhatDreamsMayCome,{roles:['AlbertLewis'],year:1998});
actsIn.save(AnnabellaS,WhatDreamsMayCome,{roles:['AnnieCollins-Nielsen'],year:1998});
actsIn.save(MaxS,WhatDreamsMayCome,{roles:['TheTracker'],year:1998});
actsIn.save(WernerH,WhatDreamsMayCome,{roles:['TheFace'],year:1998});
varSnowFallingonCedars=movies.save({_key:"SnowFallingonCedars",title:'SnowFallingonCedars',released:1999,tagline:'Firs
tloveslast.Forever.'})._id;
varEthanH=actors.save({_key:"EthanH",name:'EthanHawke',born:1970})._id;
varRickY=actors.save({_key:"RickY",name:'RickYune',born:1971})._id;
varJamesC=actors.save({_key:"JamesC",name:'JamesCromwell',born:1940})._id;
actsIn.save(EthanH,SnowFallingonCedars,{roles:['IshmaelChambers'],year:1999});
actsIn.save(RickY,SnowFallingonCedars,{roles:['KazuoMiyamoto'],year:1999});
actsIn.save(MaxS,SnowFallingonCedars,{roles:['NelsGudmundsson'],year:1999});
actsIn.save(JamesC,SnowFallingonCedars,{roles:['JudgeFielding'],year:1999});
varYouveGotMail=movies.save({_key:"YouveGotMail",title:"You'veGotMail",released:1998,tagline:'Atoddsinlife...inlov
eon-line.'})._id;
varParkerP=actors.save({_key:"ParkerP",name:'ParkerPosey',born:1968})._id;
varDaveC=actors.save({_key:"DaveC",name:'DaveChappelle',born:1973})._id;
varSteveZ=actors.save({_key:"SteveZ",name:'SteveZahn',born:1967})._id;
varTomH=actors.save({_key:"TomH",name:'TomHanks',born:1956})._id;
actsIn.save(TomH,YouveGotMail,{roles:['JoeFox'],year:1998});
actsIn.save(MegR,YouveGotMail,{roles:['KathleenKelly'],year:1998});
actsIn.save(GregK,YouveGotMail,{roles:['FrankNavasky'],year:1998});
actsIn.save(ParkerP,YouveGotMail,{roles:['PatriciaEden'],year:1998});
actsIn.save(DaveC,YouveGotMail,{roles:['KevinJackson'],year:1998});
actsIn.save(SteveZ,YouveGotMail,{roles:['GeorgePappas'],year:1998});
varSleeplessInSeattle=movies.save({_key:"SleeplessInSeattle",title:'SleeplessinSeattle',released:1993,tagline:'Whatif
someoneyounevermet,someoneyouneversaw,someoneyouneverknewwastheonlysomeoneforyou?'})._id;
varRitaW=actors.save({_key:"RitaW",name:'RitaWilson',born:1956})._id;
varBillPull=actors.save({_key:"BillPull",name:'BillPullman',born:1953})._id;
varVictorG=actors.save({_key:"VictorG",name:'VictorGarber',born:1949})._id;
varRosieO=actors.save({_key:"RosieO",name:"RosieO'Donnell",born:1962})._id;
actsIn.save(TomH,SleeplessInSeattle,{roles:['SamBaldwin'],year:1993});
actsIn.save(MegR,SleeplessInSeattle,{roles:['AnnieReed'],year:1993});
actsIn.save(RitaW,SleeplessInSeattle,{roles:['Suzy'],year:1993});
actsIn.save(BillPull,SleeplessInSeattle,{roles:['Walter'],year:1993});
actsIn.save(VictorG,SleeplessInSeattle,{roles:['Greg'],year:1993});
actsIn.save(RosieO,SleeplessInSeattle,{roles:['Becky'],year:1993});
varJoeVersustheVolcano=movies.save({_key:"JoeVersustheVolcano",title:'JoeVersustheVolcano',released:1990,tagline:'Ast
oryoflove,lavaandburningdesire.'})._id;
varNathan=actors.save({_key:"Nathan",name:'NathanLane',born:1956})._id;
actsIn.save(TomH,JoeVersustheVolcano,{roles:['JoeBanks'],year:1990});
actsIn.save(MegR,JoeVersustheVolcano,{roles:['DeDe','AngelicaGraynamore','PatriciaGraynamore'],year:1990});
actsIn.save(Nathan,JoeVersustheVolcano,{roles:['Baw'],year:1990});
varWhenHarryMetSally=movies.save({_key:"WhenHarryMetSally",title:'WhenHarryMetSally',released:1998,tagline:'Atoddsin
life...inloveon-line.'})._id;
varBillyC=actors.save({_key:"BillyC",name:'BillyCrystal',born:1948})._id;
varCarrieF=actors.save({_key:"CarrieF",name:'CarrieFisher',born:1956})._id;
varBrunoK=actors.save({_key:"BrunoK",name:'BrunoKirby',born:1949})._id;
ExampleAQLQueriesforGraphs
57
actsIn.save(BillyC,WhenHarryMetSally,{roles:['HarryBurns'],year:1998});
actsIn.save(MegR,WhenHarryMetSally,{roles:['SallyAlbright'],year:1998});
actsIn.save(CarrieF,WhenHarryMetSally,{roles:['Marie'],year:1998});
actsIn.save(BrunoK,WhenHarryMetSally,{roles:['Jess'],year:1998});
Allactorswhoactedin"movie1"OR"movie2"Saywewanttofindallactorswhoactedin"TheMatrix"OR"TheDevilsAdvocate":
Firstletstrytogetallactorsforonemovie:
db._query("FORxINANY'movies/TheMatrix'actsInOPTIONS{bfs:true,uniqueVertices:'global'}RETURNx._id").toArray();
Result:
[
[
"actors/Keanu",
"actors/Hugo",
"actors/Emil",
"actors/Carrie",
"actors/Laurence"
]
]
NowwecontinuetoformaUNION_DISTINCToftwoNEIGHBORSquerieswhichwillbethesolution:
db._query("FORxINUNION_DISTINCT((FORyINANY'movies/TheMatrix'actsInOPTIONS{bfs:true,uniqueVertices:'global'}RETUR
Ny._id),(FORyINANY'movies/TheDevilsAdvocate'actsInOPTIONS{bfs:true,uniqueVertices:'global'}RETURNy._id))RETURNx"
).toArray();
[
[
"actors/Emil",
"actors/Hugo",
"actors/Carrie",
"actors/Laurence",
"actors/Keanu",
"actors/Al",
"actors/Charlize"
]
]
Allactorswhoactedinboth"movie1"AND"movie2"?Thisisalmostidenticaltothequestionabove.ButthistimewearenotintrestedinaUNIONbutinaINTERSECTION:
db._query("FORxININTERSECTION((FORyINANY'movies/TheMatrix'actsInOPTIONS{bfs:true,uniqueVertices:'global'}RETURN
y._id),(FORyINANY'movies/TheDevilsAdvocate'actsInOPTIONS{bfs:true,uniqueVertices:'global'}RETURNy._id))RETURNx")
.toArray();
[
[
"actors/Keanu"
]
]
Allcommonmoviesbetween"actor1"and"actor2"?
ExampleAQLQueriesforGraphs
58
Thisisactuallyidenticaltothequestionaboutcommonactorsinmovie1andmovie2.Wejusthavetochangethestartingvertices.Asanexamplelet'sfindallmovieswhereHugoWeaving("Hugo")andKeanuReevesareco-starring:
db._query("FORxININTERSECTION((FORyINANY'actors/Hugo'actsInOPTIONS{bfs:true,uniqueVertices:'global'}RETURNy._id
),(FORyINANY'actors/Keanu'actsInOPTIONS{bfs:true,uniqueVertices:'global'}RETURNy._id))RETURNx").toArray();
[
[
"movies/TheMatrixRevolutions",
"movies/TheMatrixReloaded",
"movies/TheMatrix"
]
]
Allactorswhoactedin3ormoremovies?
Thisquestionisdifferent,wecannotmakeuseoftheneighborsfunctionhere.Insteadwewillmakeuseoftheedge-indexandtheCOLLECTstatementofAQLforgrouping.ThebasicideaistogroupalledgesbytheirstartVertex(whichinthisdatasetisalwaystheactor).Thenweremoveallactorswithlessthan3moviesfromtheresult.AsIamalsointerestedinthenumberofmoviesanactorhasactedin,Iincludedthevalueintheresultaswell:
db._query("FORxINactsInCOLLECTactor=x._fromWITHCOUNTINTOcounterFILTERcounter>=3RETURN{actor:actor,movies:co
unter}").toArray()
[
{
"actor":"actors/Carrie",
"movies":3
},
{
"actor":"actors/CubaG",
"movies":4
},
{
"actor":"actors/Hugo",
"movies":3
},
{
"actor":"actors/Keanu",
"movies":4
},
{
"actor":"actors/Laurence",
"movies":3
},
{
"actor":"actors/MegR",
"movies":5
},
{
"actor":"actors/TomC",
"movies":3
},
{
"actor":"actors/TomH",
"movies":3
}
]
Allmovieswhereexactly6actorsactedin?Thesameideaasinthequerybefore,butwithequalityfilter,howevernowweneedthemovieinsteadoftheactor,sowereturnthe_toattribute:
ExampleAQLQueriesforGraphs
59
db._query("FORxINactsInCOLLECTmovie=x._toWITHCOUNTINTOcounterFILTERcounter==6RETURNmovie").toArray()
[
"movies/SleeplessInSeattle",
"movies/TopGun",
"movies/YouveGotMail"
]
Thenumberofactorsbymovie?Werememberinourdataset_toontheedgecorrespondstothemovie,sowecounthowoftenthesame_toappears.Thisisthenumberofactors.ThequeryisalmostidenticaltotheonesbeforebutwithouttheFILTERafterCOLLECT:
db._query("FORxINactsInCOLLECTmovie=x._toWITHCOUNTINTOcounterRETURN{movie:movie,actors:counter}").toArray()
[
{
"movie":"movies/AFewGoodMen",
"actors":11
},
{
"movie":"movies/AsGoodAsItGets",
"actors":4
},
{
"movie":"movies/JerryMaguire",
"actors":9
},
{
"movie":"movies/JoeVersustheVolcano",
"actors":3
},
{
"movie":"movies/SleeplessInSeattle",
"actors":6
},
{
"movie":"movies/SnowFallingonCedars",
"actors":4
},
{
"movie":"movies/StandByMe",
"actors":7
},
{
"movie":"movies/TheDevilsAdvocate",
"actors":3
},
{
"movie":"movies/TheMatrix",
"actors":5
},
{
"movie":"movies/TheMatrixReloaded",
"actors":4
},
{
"movie":"movies/TheMatrixRevolutions",
"actors":4
},
{
"movie":"movies/TopGun",
"actors":6
},
{
"movie":"movies/WhatDreamsMayCome",
"actors":5
},
{
ExampleAQLQueriesforGraphs
60
"movie":"movies/WhenHarryMetSally",
"actors":4
},
{
"movie":"movies/YouveGotMail",
"actors":6
}
]
Thenumberofmoviesbyactor?Ithinkyougetthepicturebynow;)
db._query("FORxINactsInCOLLECTactor=x._fromWITHCOUNTINTOcounterRETURN{actor:actor,movies:counter}").toArray()
[
{
"actor":"actors/Al",
"movies":1
},
{
"actor":"actors/AnnabellaS",
"movies":1
},
{
"actor":"actors/AnthonyE",
"movies":1
},
{
"actor":"actors/BillPull",
"movies":1
},
{
"actor":"actors/BillyC",
"movies":1
},
{
"actor":"actors/BonnieH",
"movies":1
},
{
"actor":"actors/BrunoK",
"movies":1
},
{
"actor":"actors/Carrie",
"movies":3
},
{
"actor":"actors/CarrieF",
"movies":1
},
{
"actor":"actors/Charlize",
"movies":1
},
{
"actor":"actors/ChristopherG",
"movies":1
},
{
"actor":"actors/CoreyF",
"movies":1
},
{
"actor":"actors/CubaG",
"movies":4
},
{
"actor":"actors/DaveC",
"movies":1
ExampleAQLQueriesforGraphs
61
},
{
"actor":"actors/DemiM",
"movies":1
},
{
"actor":"actors/Emil",
"movies":1
},
{
"actor":"actors/EthanH",
"movies":1
},
{
"actor":"actors/GregK",
"movies":2
},
{
"actor":"actors/HelenH",
"movies":1
},
{
"actor":"actors/Hugo",
"movies":3
},
{
"actor":"actors/JackN",
"movies":2
},
{
"actor":"actors/JamesC",
"movies":1
},
{
"actor":"actors/JamesM",
"movies":1
},
{
"actor":"actors/JayM",
"movies":1
},
{
"actor":"actors/JerryO",
"movies":2
},
{
"actor":"actors/JohnC",
"movies":1
},
{
"actor":"actors/JonathanL",
"movies":1
},
{
"actor":"actors/JTW",
"movies":1
},
{
"actor":"actors/Keanu",
"movies":4
},
{
"actor":"actors/KellyM",
"movies":1
},
{
"actor":"actors/KellyP",
"movies":1
},
{
"actor":"actors/KevinB",
"movies":1
},
{
"actor":"actors/KevinP",
"movies":1
ExampleAQLQueriesforGraphs
62
},
{
"actor":"actors/KieferS",
"movies":2
},
{
"actor":"actors/Laurence",
"movies":3
},
{
"actor":"actors/MarshallB",
"movies":1
},
{
"actor":"actors/MaxS",
"movies":2
},
{
"actor":"actors/MegR",
"movies":5
},
{
"actor":"actors/Nathan",
"movies":1
},
{
"actor":"actors/NoahW",
"movies":1
},
{
"actor":"actors/ParkerP",
"movies":1
},
{
"actor":"actors/ReginaK",
"movies":1
},
{
"actor":"actors/ReneeZ",
"movies":1
},
{
"actor":"actors/RickY",
"movies":1
},
{
"actor":"actors/RitaW",
"movies":1
},
{
"actor":"actors/RiverP",
"movies":1
},
{
"actor":"actors/Robin",
"movies":1
},
{
"actor":"actors/RosieO",
"movies":1
},
{
"actor":"actors/SteveZ",
"movies":1
},
{
"actor":"actors/TomC",
"movies":3
},
{
"actor":"actors/TomH",
"movies":3
},
{
"actor":"actors/TomS",
"movies":1
ExampleAQLQueriesforGraphs
63
},
{
"actor":"actors/ValK",
"movies":1
},
{
"actor":"actors/VictorG",
"movies":1
},
{
"actor":"actors/WernerH",
"movies":1
},
{
"actor":"actors/WilW",
"movies":1
}
]
Thenumberofmoviesactedinbetween2005and2010byactor?ThisqueryiswhereaMultiModeldatabaseactuallyshines.Firstofallwewanttouseitinproduction,sowesetaskiplistindexonyear.Thisallowsastoexecutefastrangequerieslikebetween2005and2010.
db.actsIn.ensureSkiplist("year")
Nowweslightlymodifyourmoviesbyactorquery.Howevermydatasetcontainsonlyoldermovies,soIchangedtheyearrangefrom1990-1995:
db._query("FORxINactsInFILTERx.year>=1990&&x.year<=1995COLLECTactor=x._fromWITHCOUNTINTOcounterRETURN{acto
r:actor,movies:counter}").toArray()
[
{
"actor":"actors/BillPull",
"movies":1
},
{
"actor":"actors/ChristopherG",
"movies":1
},
{
"actor":"actors/CubaG",
"movies":1
},
{
"actor":"actors/DemiM",
"movies":1
},
{
"actor":"actors/JackN",
"movies":1
},
{
"actor":"actors/JamesM",
"movies":1
},
{
"actor":"actors/JTW",
"movies":1
},
{
"actor":"actors/KevinB",
"movies":1
},
{
"actor":"actors/KevinP",
"movies":1
},
ExampleAQLQueriesforGraphs
64
{
"actor":"actors/KieferS",
"movies":1
},
{
"actor":"actors/MegR",
"movies":2
},
{
"actor":"actors/Nathan",
"movies":1
},
{
"actor":"actors/NoahW",
"movies":1
},
{
"actor":"actors/RitaW",
"movies":1
},
{
"actor":"actors/RosieO",
"movies":1
},
{
"actor":"actors/TomC",
"movies":1
},
{
"actor":"actors/TomH",
"movies":2
},
{
"actor":"actors/VictorG",
"movies":1
}
]
CommentAuthor:MichaelHackstein
Tags:#graph#examples
ExampleAQLQueriesforGraphs
65
UseCases/ExamplesCrawlingGithubwithPromises
UsingArangoDBwithSails.js
PopulatingaTextbox
ExportingData
AccessingbasedocumentswithJava
AddXMLdatatoArangoDBwithJava
UseCases/Examples
66
CrawlingGithubwithPromises
Problem
ThenewArangoDBJavascriptdrivernolongerimposesanypromisesimplementation.Itfollowsthestandardcallbackpatternwithacallbackusingerrandres.
Butwhatifwewanttouseapromiselibrary-inthiscasethemostpopularonepromises?LetsgiveitatryandbuildagithubcrawlerwiththenewJavascriptdriverandpromises.
SolutionThefollowingsourcecodecanbefoundongithub.
PaginationwithPromisesmadeeasy
Thegithubdriverhasafunctiontogetallfollowers.However,theresultispaginated.Withtwohelperfunctionsandpromisesitisstraightforwardtoimplementafunctiontoretrieveallfollowersofanuser.
functionextractFollowers(name){
'usestrict';
returnnewPromise(function(resolve,reject){
github.user.getFollowers({user:name},promoteError(reject,function(res){
followPages(resolve,reject,[],res);
}));
});
}
ThefollowPagesfunctionsimplyextendstheresultwiththenextpageuntilthelastpageisreached.
functionfollowPages(resolve,reject,result,res){
'usestrict';
vari;
for(i=0;i<res.length;++i){
result.push(res[i]);
}
if(github.hasNextPage(res)){
github.getNextPage(res,promoteError(reject,function(res){
followPages(resolve,reject,result,res);
}));
}
else{
resolve(result);
}
}
Thepromoteerrorhelperisaconveniencefunctiontobridgecallbacksandpromises.
functionpromoteError(reject,resolve){
'usestrict';
returnfunction(err,res){
if(err){
if(err.hasOwnProperty("message")&&/ratelimitexceeded/.test(err.message)){
rateLimitExceeded=true;
}
console.error("caughterror:%s",err);
reject(err);
CrawlingGithubwithPromises
67
}
else{
resolve(res);
}
};
}
I'vedecidedtosticktothesequencereject(akaerr)followedbyresolve(akares)-likethecallbacks.ThepromoteErrorcanbeusedforthegithubcallbackaswellastheArangoDBdriver.
Queues,Queues,Queues
I'veonlyneededaverysimplejobqueue,soqueue-itisagoodchoice.ItprovidesaverysimpleAPIforhandlingjobqueues:
POST/queue/job
POST/queue/worker
DELETE/queue/job/:key
ThenewJavascriptdriverallowstoaccessarbitraryendpoint.FirstinstallaFoxximplementingthequeuemicroserviceinanArangoDBinstance.
foxx-managerinstallqueue-it/queue
Addinganewjobfromnode.jsisnoweasy
functionaddJob(data){
'usestrict';
returnnewPromise(function(resolve,reject){
db.endpoint("queue").post("job",data,promoteError(reject,resolve));
});
}
Transaction
Iwantedtocrawlusersandtheirrepos.Therelations("follows","owns","is_member","stars")isstoredinanedgecollection.Ionlyaddanedgeifitisnotalreadythere.ThereforeIcheckinsideatransaction,iftheedgeexistsandaddit,ifitdoesnot.
createRepoDummy(repo.full_name,data).then(function(dummyData){
returndb.transaction(
"relations",
String(function(params){
varme=params[0];
varyou=params[1];
vartype=params[2];
vardb=require("org/arangodb").db;
if(db.relations.firstExample({_from:me,_to:you,type:type})===null){
db.relations.save(me,you,{type:type});
}
}),
[meId,"repos/"+data._key,type],
function(err){
if(err){
throwerr;
}
returnhandleDummy(dummyData);
});
})
Pleasenotethattheactionfunctionisexecutedontheserverandnotinthenodejsclient.Thereforeweneedtopasstherelevantdataasparameters.Itisnotpossibletousetheclosurevariables.
RidingtheBeast
CrawlingGithubwithPromises
68
StartanArangoDBinstance(i.e.insideadockercontainer)andinstallthesimplequeue.
foxx-managerinstallqueue-it/queue
Startthearangoshandcreatecollectionsusers,reposandrelations.
arangosh>db._create("users");
arangosh>db.users.ensureHashIndex("name");
arangosh>db._create("repos");
arangosh>db.users.ensureHashIndex("name");
arangosh>db._createEdgeCollection("relations");
Noweverythingisinitialized.Fireupnodejsandstartcrawling:
node>varcrawler=require("./crawler");
node>crawler.github.authenticate({type:"basic",username:"username",password:"password"})
node>crawler.addJob({type:"user",identifier:"username"})
node>crawler.runJobs();
Comment
Pleasekeepinmindthatthisisjustanexperiment.Thereisnogooderrorhandlingandconveniencefunctionsforsetupandstart.Itisalsonotoptimizedforperformance.Forinstance,itwouldeasilybepossibletoavoidnodejs/ArangoDBroundtripsusingmoretransactions.
Sourcesusedinthisexample:
ArangoJSnpmpromisesArangoDBFoxxqueue-it
ThesourcecodeofthisexampleisavailablefromGithub:https://github.com/fceller/Foxxmender
Author:FrankCeller
Tags:#foxx#javascript#API#nodejs#driver
CrawlingGithubwithPromises
69
HowtouseArangoDBwithSails.jsFirstinstalltheSails.jsframeworkusingNPM:
npminstall-gsails
NowyoucancreateanewSails.jsappnamedsomenamewiththefollowingcommand:
sailsnewsomename
NowweneedtoaddArangoDBtothisnewapplication.Firstcdintothefreshlycreatedsomenamedirectory.TheninstalltheArangoDBadapterforSails.jswiththefollowingcommand:
npminstallsails-arangodb
Thishoweveronlyinstallsthenecessarydependency.Weneedtoconfiguretheapplicationtoloadtheadapter.Openthefileconfig/connections.js.Youwillseealistofexampleconfigurationsforpossibleconnectionstodatabases.Removetheonesyoudon'tneed(orjustkeepallofthem),andaddthefollowingconfiguration(adjustthehost,port,databasenameandgraphnametoyourneeds):
localArangoDB:{
adapter:'sails-arangodb',
host:'localhost',
port:8529,
database:{
name:'sails',
graph:'sails'
}
}
Now,youcanconfigureyourapptousetheArangoDBasyourdefaultconnection.Youdothisbyadjustingthefileconfig/models.js:
module.exports.models={
connection:'localArangoDB'//thisisthenamefromtheconnections.jsfile
//...
};
YourappisnowconfiguredtouseArangoDBforallmodelsbydefault.Youcannowforexamplecreateablueprintcontrollerbytypingthefollowinginyourconsole:
sailsgenerateapitodos
Nowyoucanbootyourapplicationwith:
sailslift
Youcannowaccesshttp://localhost:1337/todosandseeanemptylistoftodos.Andthencreateatodobyvisitinglocalhost:1337/todos/create?name=john.Thiswillcreatetheaccordingdocument(thathasanattributenamewiththevaluejohn)inthetodoscollectionoftheselecteddatabase.Youwillalsoseethedocumentwhenyouvisithttp://localhost:1337/todosagain.
Author:LucasDohmen
Tags:#nodejs
UsingArangoDBwithSails.js
70
Populatinganautocompletetextbox
Problem
Iwanttopopulateanautocompletetextboxwithvaluesfromacollection.Thecompletionsshouldadjustdynamicallybasedonuserinput.
Solution
Useawebframeworkfortheclient-sideautocompleterenderingandeventprocessing.Useacollectionwitha(sorted)skiplistindexandarangequeryonittoefficientlyfetchthecompletionvaluesdynamically.ConnectthetwousingasimpleFoxxroute.
Installanexampleapp
Thisappcontainsajquery-poweredwebpagewithanautocompletetextbox.Itusesjqueryautocomplete,buteveryotherwebframeworkwillalsodo.
Theappcanbeinstalledasfollows:
intheArangoDBwebinterface,switchintotheApplicationstabthere,clickAddApplicationswitchontheGithubtabforRepository,enterjsteemann/autocompleteforVersion,entermasterclickInstall
Nowenteramountpointfortheapplication.ThisistheURLpathunderwhichtheapplicationwillbecomeavailable.Fortheexampleapp,themountpointdoesnotmatter.ThewebpageintheexampleappassumesitisservedbyArangoDB,too.SoitusesarelativeURLautocomplete.Thisiseasiesttosetup,butinrealityyoumightwanttohaveyourwebpageservedbyadifferentserver.Inthiscase,yourwebpagewillhavetocalltheappmountpointyoujustentered.
Toseetheexampleappinaction,clickonOpen.Theautocompletetextboxshouldbepopulatedwithserverdatawhenatleasttwolettersareentered.
Backendcode,setupscript
Theappalsocontainsabackendroute/autocompletewhichiscalledbythewebpagetofetchcompletionsbasedonuserinput.TheHTMLcodeforthewebpageishere.
Containedintheappisasetupscriptthatwillcreateacollectionnamedcompletionsandloadsomeinitialdataintoit.TheexampleappprovidesautocompletionforUScitynames,andthesetupscriptpopulatesthecollectionwithabout10Kcitynames.
Thesetupscriptalsocreatesaskiplistindexonthelookupattribute,sothisattributecanbeusedforefficientfilteringandsortinglater.Thelookupattributecontainsthecitynamesalreadylower-cased,andtheoriginal(pretty)namesarestoredinattributepretty.Thisattributewillbereturnedtousers.
Backendcode,Foxxroutecontroller
Theappcontainsacontroller.Thebackendaction/autocompletethatiscalledbythewebpageisalsocontainedherein:
controller.get("/autocomplete",function(req,res){
//searchphraseenteredbyuser
varsearchString=req.params("q").trim()||"";
//lowerboundforsearchrange
varbegin=searchString.replace(/[^a-zA-Z]/g,"").toLowerCase();
if(begin.length===0){
//searchphraseisempty-noneedtoperfomasearchatall
res.json([]);
PopulatingaTextbox
71
return;
}
//upperboundforsearchrange
varend=begin.substr(0,begin.length-1)+String.fromCharCode(begin.charCodeAt(begin.length-1)+1);
//bindparametersforquery
varqueryParams={
"@collection":"completions",
"begin":begin,
"end":end
};
//thesearchquery
varquery="FORdocIN@@collectionFILTERdoc.lookup>=@begin&&doc.lookup<@endSORTdoc.lookupRETURN{label:doc.pre
tty,value:doc.pretty,id:doc._key}";
res.json(db._query(query,queryParams).toArray());
}
ThebackendcodefirstfetchesthesearchstringfromtheURLparameterq.Thisiswhatthewebpagewillsendus.
Basedonthesearchstring,alookuprangeiscalculated.Firstofall,thesearchstringislower-casedandallnon-lettercharactersareremovedfromit.Theresultingstringisthelowerboundforthelookup.Fortheupperbound,wecanusethelowerboundwithitslastlettercharactercodeincreasedbyone.
Forexample,iftheuserenteredLosAintothetextbox,thewebpagewillsendusthestringLosAinURLparameterq.Lower-casingandremovingnon-lettercharactersfromthestring,we'llgetlosa.Thisisthelowerbound.Theupperboundislosa,withitslastletteradjustedtob(i.e.losb).
Finally,thelowerandupperboundsareinsertedintothefollowingqueryusingbindparameters@beginand@end:
FORdocIN@@collection
FILTERdoc.lookup>=@begin&&doc.lookup<@end
SORTdoc.lookup
RETURN{
label:doc.pretty,
value:doc.pretty,
id:doc._key
}
Thecitynamesinthelookuprangewillbereturnedsorted.Foreachcity,threevaluesarereturned(theidcontainsthedocumentkey,theothertwovaluesarefordisplaypurposes).Otherframeworksmayrequireadifferentreturnformat,butthatcaneasilybedonebyadjustingtheAQLquery.
Author:JanSteemann
Tags:#aql#autocomplete#jquery
PopulatingaTextbox
72
ExportingDataforOfflineProcessingInthisrecipewewilllearnhowtousetheexportAPItoextractdataandprocessitwithPHP.AttheendoftherecipeyoucandownloadthecompletePHPscript.
Note:ThefollowingrecipeiswrittenusinganArangoDBserverwithversion2.6orhigher.Youcanalsousethedevelbranch,sinceversion2.6hasn'tbeenanofficialreleaseyet.
Howto
Importingexampledata
FirstofallweneedsomedatainanArangoDBcollection.Forthisexamplewewilluseacollectionnameduserswhichwewillpopulatewith100.000exampledocuments.ThiswayyoucangetthedataintoArangoDB:
#downloaddatafile
wgethttps://jsteemann.github.io/downloads/code/users-100000.json.tar.gz
#uncompressit
tarxvfzusers-100000.json.tar.gz
#importintoArangoDB
arangoimp--fileusers-100000.json--collectionusers--create-collectiontrue
SettingupArangoDB-PHP
ForthisrecipewewillusetheArangoDBPHPdriver:
gitclone-bdevel"https://github.com/arangodb/arangodb-php.git"
WewillnowwriteasimplePHPscriptthatestablishesaconnectiontoArangoDBonlocalhost:
<?php
namespacetriagens\ArangoDb;
//usethedriver'sautoloadertoloadclasses
require'arangodb-php/autoload.php';
Autoloader::init();
//setupconnectionoptions
$connectionOptions=array(
//endpointtoconnectto
ConnectionOptions::OPTION_ENDPOINT=>'tcp://localhost:8529',
//canuseKeep-Aliveconnection
ConnectionOptions::OPTION_CONNECTION=>'Keep-Alive',
//usebasicauthorization
ConnectionOptions::OPTION_AUTH_TYPE=>'Basic',
//userforbasicauthorization
ConnectionOptions::OPTION_AUTH_USER=>'root',
//passwordforbasicauthorization
ConnectionOptions::OPTION_AUTH_PASSWD=>'',
//timeoutinseconds
ConnectionOptions::OPTION_TIMEOUT=>30,
//databasename
ConnectionOptions::OPTION_DATABASE=>'_system'
);
try{
//establishconnection
$connection=newConnection($connectionOptions);
echo'Connected!'.PHP_EOL;
//TODO:nowdosomethingusefulwiththeconnection!
ExportingData
73
}catch(ConnectException$e){
print$e.PHP_EOL;
}catch(ServerException$e){
print$e.PHP_EOL;
}catch(ClientException$e){
print$e.PHP_EOL;
}
AfterrunningthescriptyoushouldseeConnected!inthebashifsuccessful.
Extractingthedata
Nowwecanrunanexportofthedatainthecollectionusers.PlacethefollowingcodeintotheTODOpartofthefirstcode:
functionexport($collection,Connection$connection){
$fp=fopen('output.json','w');
if(!$fp){
thrownewException('couldnotopenoutputfile!');
}
//settingstousefortheexport
$settings=array(
'batchSize'=>5000,//exportinchunksof5Kdocuments
'_flat'=>true//usesimplePHParrays
);
$export=newExport($connection,$collection,$settings);
//executetheexport.thiswillreturnanexportcursor
$cursor=$export->execute();
//statistics
$count=0;
$batches=0;
$bytes=0;
//nowwecanfetchthedocumentsfromthecollectioninbatches
while($docs=$cursor->getNextBatch()){
$output='';
foreach($docsas$doc){
$output.=json_encode($doc).PHP_EOL;
}
//writeoutchunk
fwrite($fp,$output);
//updatestatistics
$count+=count($docs);
$bytes+=strlen($output);
++$batches;
}
fclose($fp);
echosprintf('written%ddocumentsin%dbatcheswith%dtotalbytes',
$count,
$batches,
$bytes).PHP_EOL;
}
//runtheexport
export('users',$connection);
Thefunctionextractsalldocumentsfromthecollectionandwritesthemintoanoutputfileoutput.json.Inadditionitwillprintsomestatisticsaboutthenumberofdocumentsandthetotaldatasize:
written100000documentsin20batcheswith40890013totalbytes
Applyingsometransformations
ExportingData
74
WenowwillusePHPtotransformdataasweextractit:
functiontransformDate($value){
returnpreg_replace('/^(\\d+)-(\\d+)-(\\d+)$/','\\2/\\3/\\1',$value);
}
functiontransform(array$document){
static$genders=array('male'=>'m','female'=>'f');
$transformed=array(
'gender'=>$genders[$document['gender']],
'dob'=>transformDate($document['birthday']),
'memberSince'=>transformDate($document['memberSince']),
'fullName'=>$document['name']['first'].''.$document['name']['last'],
'email'=>$document['contact']['email'][0]
);
return$transformed;
}
functionexport($collection,Connection$connection){
$fp=fopen('output-transformed.json','w');
if(!$fp){
thrownewException('couldnotopenoutputfile!');
}
//settingstousefortheexport
$settings=array(
'batchSize'=>5000,//exportinchunksof5Kdocuments
'_flat'=>true//usesimplePHParrays
);
$export=newExport($connection,$collection,$settings);
//executetheexport.thiswillreturnanexportcursor
$cursor=$export->execute();
//nowwecanfetchthedocumentsfromthecollectioninbatches
while($docs=$cursor->getNextBatch()){
$output='';
foreach($docsas$doc){
$output.=json_encode(transform($doc)).PHP_EOL;
}
//writeoutchunk
fwrite($fp,$output);
}
fclose($fp);
}
//runtheexport
export('users',$connection);
Withthisscriptthefollowingchangeswillbemadeonthedata:
rewritethecontentsofthegenderattribute.femalebecomesfandmalebecomesmbirthdaynowbecomesdobthedateformationswillbechangedfromYYYY-MM-DDtoMM/DD/YYYYconcatenatethecontentsofname.firstandname.lastcontact.emailwillbetransformedfromanarraytoaflatstringeveryotherattributewillberemoved
Note:Theoutputwillbeinafilenamedoutput-transformed.json.
Filteringattributes
Excludecertainattributes
ExportingData
75
Insteadoffilteringoutasdoneinthepreviousexamplewecaneasilyconfiguretheexporttoexcludetheseattributesserver-side:
//settingstousefortheexport
$settings=array(
'batchSize'=>5000,//exportinchunksof5Kdocuments
'_flat'=>true,//usesimplePHParrays
'restrict'=>array(
'type'=>'exclude',
'fields'=>array('_id','_rev','_key','likes')
)
);
Thisscriptwillexcludetheattributes_id,_rev._keyandlikes.
Includecertainattributes
Wecanalsoincludeattributeswiththefollowingscript:
functionexport($collection,Connection$connection){
//settingstousefortheexport
$settings=array(
'batchSize'=>5000,//exportinchunksof5Kdocuments
'_flat'=>true,//usesimplePHParrays
'restrict'=>array(
'type'=>'include',
'fields'=>array('_key','name')
)
);
$export=newExport($connection,$collection,$settings);
//executetheexport.thiswillreturnanexportcursor
$cursor=$export->execute();
//nowwecanfetchthedocumentsfromthecollectioninbatches
while($docs=$cursor->getNextBatch()){
$output='';
foreach($docsas$doc){
$values=array(
$doc['_key'],
$doc['name']['first'].''.$doc['name']['last']
);
$output.='"'.implode('","',$values).'"'.PHP_EOL;
}
//printoutthedatadirectly
print$output;
}
}
//runtheexport
export('users',$connection);
Inthisscriptonlythe_keyandnameattributesareextracted.Intheprintsthe_key/namepairsareinCSVformat.
Note:Thewholescriptcanbedownloaded.
UsingtheAPIwithoutPHP
TheexportAPIRESTinterfacecanbeusedwithanyclientthatcanspeakHTTPlikecurl.Withthefollowingcommandyoucanfetchthedocumentsfromtheuserscollection:
curl
-XPOST
http://localhost:8529/_api/export?collection=users
--data'{"batchSize":5000}'
ExportingData
76
TheHTTPresponsewillcontatinaresultattributethatcontainstheactualdocuments.TheattributehasMorewillindicateiftherearemoredocumentsfortheclienttofetch.TheHTTPwillalsocontainanattributeidifsettotrue.
Withtheidyoucansendfollow-uprequestslikethis:
curl
-XPUT
http://localhost:8529/_api/export/13979338067709
Authors:ThomasSchmidtsandJanSteemann
Tags:#howto#php
ExportingData
77
HowtoretrievedocumentsfromArangoDBwithoutknowingthestructure?
Problem
IfyouuseaNoSQLdatabaseit'scommontoretrievedocumentswithanunknownattributestructure.Furthermore,theamountandtypesofattributesmaydifferindocumentsresultingfromasinglequery.Anotherproblemisthatyouwanttoaddoneoremoreattributestoadocument.
InJavayouareusedtoworkwithobjects.Regardingtheupperrequirementsitispossibletodirectlyretrieveobjectswiththesameattributestructureasthedocumentoutofthedatabase.Addingattributestoanobjectatruntimecouldbeverymessy.
Note:ArangoDB3.1andthecorrespondingJavadriverisneeded.
Solution
WiththelatestversionoftheJavadriverofArangoDBanobjectcalledBaseDocumentisprovided.
Thestructureisverysimple:Itonlyhasfourattributes:
publicclassBaseDocument{
Stringid;
Stringkey;
Stringrevision;
Map<String,Object>properties;
}
Thefirstthreeattributesarethesystemattributes_id,_keyand_rev.ThefourthattributeisaHashMap.ThekeyalwaysisaString,thevalueanobject.Thesepropertiescontainallnonsystemattributesofthedocument.
Themapcancontainvaluesofthefollowingtypes:
Map
ListBooleanNumberStringnull
Note:MapandListcontainobjects,whichareofthesametypesaslistedabove.
Toretrieveadocumentissimilartotheknownprocedure,exceptthatyouuseBaseDocumentastype.
ArangoDB.Builderarango=newArangoDB.Builder().builder();
DocumentEntity<BaseDocument>myObject=arango.db().collection("myCollection").getDocument("myDocumentKey",BaseDocument.class)
;
Otherresources
AccessingbasedocumentswithJava
78
MoredocumentationabouttheArangoDBJavadriverisavailable:
Tutorial:JavaintenminutesJavadriveratGithubExamplesourcecodeJavaDoc
Author:gschwab,MarkVollmary
Tags:#java#driver
AccessingbasedocumentswithJava
79
HowtoaddXMLdatatoArangoDB?
Problem
YouwanttostoreXMLdatafilesintoadatabasetohavetheabilitytomakequeriesontothem.
Note:ArangoDB3.1andthecorrespondingJavadriverisneeded.
SolutionSinceversion3.1.0thearagodb-java-driversupportswriting,readingandqueryingofrawstringscontainingtheJSONdocuments.
WithJsonMLyoucanconvertaXMLstringintoaJSONstringandbacktoXMLagain.
ConvertingXMLintoJSONwithJsonMLexample:
Stringstring="<recipename=\"bread\"prep_time=\"5mins\"cook_time=\"3hours\">"
+"<title>Basicbread</title>"
+"<ingredientamount=\"8\"unit=\"dL\">Flour</ingredient>"
+"<ingredientamount=\"10\"unit=\"grams\">Yeast</ingredient>"
+"<ingredientamount=\"4\"unit=\"dL\"state=\"warm\">Water</ingredient>"
+"<ingredientamount=\"1\"unit=\"teaspoon\">Salt</ingredient>"
+"<instructions>"
+"<step>Mixallingredientstogether.</step>"
+"<step>Kneadthoroughly.</step>"
+"<step>Coverwithacloth,andleaveforonehourinwarmroom.</step>"
+"<step>Kneadagain.</step>"
+"<step>Placeinabreadbakingtin.</step>"
+"<step>Coverwithacloth,andleaveforonehourinwarmroom.</step>"
+"<step>Bakeintheovenat180(degrees)Cfor30minutes.</step>"
+"</instructions>"
+"</recipe>";
JSONObjectjsonObject=JSONML.toJSONObject(string);
System.out.println(jsonObject.toString());
TheconvertedJSONstring:
{
"prep_time":"5mins",
"name":"bread",
"cook_time":"3hours",
"tagName":"recipe",
"childNodes":[
{
"childNodes":[
"Basicbread"
],
"tagName":"title"
},
{
"childNodes":[
"Flour"
],
"tagName":"ingredient",
"amount":8,
"unit":"dL"
},
{
"unit":"grams",
"amount":10,
"tagName":"ingredient",
"childNodes":[
"Yeast"
]
},
AddXMLdatatoArangoDBwithJava
80
{
"childNodes":[
"Water"
],
"tagName":"ingredient",
"amount":4,
"unit":"dL",
"state":"warm"
},
{
"childNodes":[
"Salt"
],
"tagName":"ingredient",
"unit":"teaspoon",
"amount":1
},
{
"childNodes":[
{
"tagName":"step",
"childNodes":[
"Mixallingredientstogether."
]
},
{
"tagName":"step",
"childNodes":[
"Kneadthoroughly."
]
},
{
"childNodes":[
"Coverwithacloth,andleaveforonehourinwarmroom."
],
"tagName":"step"
},
{
"tagName":"step",
"childNodes":[
"Kneadagain."
]
},
{
"childNodes":[
"Placeinabreadbakingtin."
],
"tagName":"step"
},
{
"tagName":"step",
"childNodes":[
"Coverwithacloth,andleaveforonehourinwarmroom."
]
},
{
"tagName":"step",
"childNodes":[
"Bakeintheovenat180(degrees)Cfor30minutes."
]
}
],
"tagName":"instructions"
}
]
}
SavingtheconvertedJSONtoArangoDBexample:
ArangoDB.Builderarango=newArangoDB.Builder().build();
ArangoCollectioncollection=arango.db().collection("testCollection")
DocumentCreateEntity<String>entity=collection.insertDocument(
jsonObject.toString());
Stringkey=entity.getKey();
AddXMLdatatoArangoDBwithJava
81
ReadingthestoredJSONasastringandconvertitbacktoXMLexample:
StringrawJsonString=collection.getDocument(key,String.class);
Stringxml=JSONML.toString(rawJsonString);
System.out.println(xml);
Exampleoutput:
<recipe_id="RawDocument/6834407522"_key="6834407522"_rev="6834407522"
cook_time="3hours"name="bread"prep_time="5mins">
<title>Basicbread</title>
<ingredientamount="8"unit="dL">Flour</ingredient>
<ingredientamount="10"unit="grams">Yeast</ingredient>
<ingredientamount="4"state="warm"unit="dL">Water</ingredient>
<ingredientamount="1"unit="teaspoon">Salt</ingredient>
<instructions>
<step>Mixallingredientstogether.</step>
<step>Kneadthoroughly.</step>
<step>Coverwithacloth,andleaveforonehourinwarmroom.</step>
<step>Kneadagain.</step>
<step>Placeinabreadbakingtin.</step>
<step>Coverwithacloth,andleaveforonehourinwarmroom.</step>
<step>Bakeintheovenat180(degrees)Cfor30minutes.</step>
</instructions>
</recipe>
Note:ThefieldsmandatorytoArangoDBdocumentsareadded;IftheybreakyourXMLschemayouhavetoremovethem.
Queryrawdataexample:
StringqueryString="FORtINtestCollectionFILTERt.cook_time=='3hours'RETURNt";
ArangoCursor<String>cursor=arango.db().query(queryString,null,null,String.class);
while(cursor.hasNext()){
JSONObjectjsonObject=newJSONObject(cursor.next());
Stringxml=JSONML.toString(jsonObject);
System.out.println("XMLvalue:"+xml);
}
OtherresourcesMoredocumentationabouttheArangoDBJavadriverisavailable:
Tutorial:JavaintenminutesJavadriveratGithubExamplesourcecodeJavaDoc
Author:AchimBrandt,MarkVollmary
Tags:#java#driver
AddXMLdatatoArangoDBwithJava
82
AdministrationUsingAuthentication
ImportingData
ReplicatingData
XCopyInstallWindows
Migrating2.8to3.0
Administration
83
Usingauthentication
Problem
IwanttouseauthenticationinArangoDB.
Solution
Inordertomakeauthenticationworkproperly,youwillneedtocreateuseraccountsfirst.
ThenadjustArangoDB'sconfigurationandturnonauthentication(ifit'soff).
Setuporadjustuseraccounts
ArangoDBuseraccountsarevalidthroughoutaserverinstanceanduserscanbegrantedaccesstooneormoredatabases.Theyaremanagedthroughthedatabasenamed_system.
Tomanageuseraccounts,connectwiththeArangoShelltotheArangoDBhostandthe_systemdatabase:
$arangosh--server.endpointtcp://127.0.0.1:8529--server.database"_system"
Bydefault,arangoshwillconnectwithausernamerootandanemptypassword.Thiswillworkifauthenticationisturnedoff.
Whenconnected,youcancreateanewuseraccountwiththefollowingcommand:
arangosh>require("org/arangodb/users").save("myuser","mypasswd");
myuserwillbetheusernameandmypasswdwillbetheuser'spassword.NotethatrunningthecommandlikethismaystorethepasswordliterallyinArangoShell'shistory.
Toavoidthat,useadynamicallycreatedpassword,e.g.:
arangosh>passwd=require("internal").genRandomAlphaNumbers(20);
arangosh>require("org/arangodb/users").save("myuser",passwd);
Theabovewillprintthepasswordonscreen(soyoucanmemorizeit)butwon'tstoreitinthecommandhistory.
Whilethere,youprobablywanttochangethepasswordofthedefaultrootusertoo.Otherwiseonewillbeabletoconnectwiththedefaultrootuseranditsemptypassword.Thefollowingcommandschangetherootuser'spassword:
arangosh>passwd=require("internal").genRandomAlphaNumbers(20);
arangosh>require("org/arangodb/users").update("root",passwd);
Turnonauthentication
AuthenticationisturnedonbydefaultinArangoDB.Youshouldmakesurethatitwasnotturnedoffmanuallyhowever.Checktheconfigurationfile(normallynamed/etc/arangodb.conf)andmakesureitcontainsthefollowinglineintheserversection:
authentication=true
ThiswillmakeArangoDBrequireauthenticationforeveryrequest(includingrequeststoFoxxapps).
IfyouwanttorunFoxxappswithoutHTTPauthentcation,butactivateHTTPauthenticationforthebuilt-inserverAPIs,youcanaddthefollowinglineintheserversectionoftheconfiguration:
authentication-system-only=true
UsingAuthentication
84
TheabovewillbypassauthenticationforrequeststoFoxxapps.
Whenfinishedmakingchanges,youneedtorestartArangoDB:
servicearangodbrestart
Checkaccessibility
Toconfirmauthenticationisineffect,tryconnectingtoArangoDBwiththeArangoShell:
$arangosh--server.endpointtcp://127.0.0.1:8529--server.database"_system"
Theabovewillimplicityuseausernamerootandanemptypasswordwhenconnecting.Ifyouchangedthepasswordoftherootaccountasdescribedabove,thisshouldnotworkanymore.
Youshouldalsovalidatethatyoucanconnectwithavaliduser:
$arangosh--server.endpointtcp://127.0.0.1:8529--server.database"_system"--server.usernamemyuser
YoucanalsousecurltocheckthatyouareactuallygettingHTTP401(Unauthorized)serverresponsesforrequeststhatrequireauthentication:
$curl--dump-http://127.0.0.1:8529/_api/version
Author:JanSteemann
Tags:#authentication#security
UsingAuthentication
85
Importingdata
Problem
IwanttoimportdatafromafileintoArangoDB.
Solution
ArangoDBcomeswithacommand-linetoolutilitynamedarangoimp.ThisutilitycanbeusedforimportingJSON-encoded,CSV,andtab-separatedfilesintoArangoDB.
arangoimpneedstobeinvokedfromthecommand-lineonceforeachimportfile.Thetargetcollectioncanalreadyexistorcanbecreatedbytheimportrun.
ImportingJSON-encodeddata
Inputformats
TherearetwosupportedinputformatsforimportingJSON-encodeddataintoArangoDB:
line-by-lineformat:ThisformatexpectseachlineintheinputfiletobeavalidJSONobjects.NolinebreaksmustoccurwithineachsingleJSONobject
arrayformat:ExpectsafilecontainingasinglearrayofJSONobjects.WhitespaceisallowedforformattinginsidetheJSONarrayandtheJSONobjects
Here'sanexamplefortheline-by-lineformatlookslikethis:
{"author":"FrankCeller","time":"2011-10-2608:42:49+0200","sha":"c413859392a45873936cbe40797970f8eed93ff9","message":"firstc
ommit","user":"f.celler"}
{"author":"FrankCeller","time":"2011-10-2621:32:36+0200","sha":"10bb77b8cc839201ff59a778f0c740994083c96e","message":"initial
release","user":"f.celler"}
...
Here'sanexampleforthesamedatainarrayformat:
[
{
"author":"FrankCeller",
"time":"2011-10-2608:42:49+0200",
"sha":"c413859392a45873936cbe40797970f8eed93ff9",
"message":"firstcommit",
"user":"f.celler"
},
{
"author":"FrankCeller",
"time":"2011-10-2621:32:36+0200",
"sha":"10bb77b8cc839201ff59a778f0c740994083c96e",
"message":"initialrelease",
"user":"f.celler"
},
...
]
ImportingJSONdatainline-by-lineformat
Anexampledatafileinline-by-lineformatcanbedownloadedhere.TheexamplefilecontainsallthecommitstotheArangoDBrepositoryasshownbygitlog--reverse.
Thefollowingcommandswillimportthedatafromthefileintoacollectionnamedcommits:
ImportingData
86
#downloadfile
wgethttp://jsteemann.github.io/downloads/code/git-commits-single-line.json
#actuallyimportdata
arangoimp--filegit-commits-single-line.json--collectioncommits--create-collectiontrue
Notethatnofiletypehasbeenspecifiedwhenarangoimpwasinvoked.Thisisbecausejsonisitsdefaultinputformat.
Theotherparametersusedhavethefollowingmeanings:
file:inputfilenamecollection:nameofthetargetcollectioncreate-collection:whetherornotthecollectionshouldbecreatedifitdoesnotexist
Theresultoftheimportprintedbyarangoimpshouldbe:
created:20039
warnings/errors:0
total:20039
Thecollectioncommitsshouldnowcontaintheexamplecommitdataaspresentintheinputfile.
ImportingJSONdatainarrayformat
Anexampleinputfileforthearrayformatcanbefoundhere.
ThecommandforimportingJSONdatainarrayformatissimilartowhatwe'vedonebefore:
#downloadfile
wgethttp://jsteemann.github.io/downloads/code/git-commits-array.json
#actuallyimportdata
arangoimp--filegit-commits-array.json--collectioncommits--create-collectiontrue
Thoughtheimportcommandisthesame(exceptthefilename),thereisanotabledifferencebetweenthetwoJSONformats:forthearrayformat,arangoimpwillreadandparsetheJSONinitsentiretybeforeitsendsanydatatotheArangoDBserver.Thatmeansthewholeinputfilemustfitintoarangoimp'sbuffer.Bydefault,arangoimpwillallocatea16MiBinternalbuffer,andinputfilesbiggerthanthatwillberejectedwiththefollowingmessage:
importfileistoobig.pleaseincreasethevalueof--batch-size(currently16777216).
SoforJSONinputfilesinarrayformatitmightbenecessarytoincreasethevalueof--batch-sizeinordertohavethefileimported.Alternatively,theinputfilecanbeconvertedtoline-by-lineformatmanually.
ImportingCSVdata
DatacanalsobeimportedfromaCSVfile.Anexamplefilecanbefoundhere.
The--typeparameterfortheimportcommandmustnowbesettocsv:
#downloadfile
wgethttp://jsteemann.github.io/downloads/code/git-commits.csv
#actuallyimportdata
arangoimp--filegit-commits.csv--typecsv--collectioncommits--create-collectiontrue
FortheCSVimport,thefirstlineintheinputfilehasaspecialmeaning:everyvaluelistedinthefirstlinewillbetreatedasanattributenameforthevaluesinallfollowinglines.Allfollowinglinesshouldalsohavethesamenumberof"columns".
"columns"insidetheCSVinputfilecanbeleftemptythough.Ifa"column"isleftemptyinaline,thenthisvaluewillbeomittedfortheimportsotherespectiveattributewillnotbesetintheimporteddocument.Notethatvaluesfromtheinputfilethatareenclosedindoublequoteswillalwaysbeimportedasstrings.Toimportnumericvalues,booleanvaluesorthenullvalue,don'tenclosethese
ImportingData
87
valuesinquotesintheinputfile.Notethatleadingzerosinnumericvalueswillberemoved.Importingnumberswithleadingzeroswillonlyworkwhenputtingthenumbersintostrings.
HereisanexampleCSVfile:
"author","time","sha","message"
"FrankCeller","2011-10-2608:42:49+0200","c413859392a45873936cbe40797970f8eed93ff9","firstcommit"
"FrankCeller","2011-10-2621:32:36+0200","10bb77b8cc839201ff59a778f0c740994083c96e","initialrelease"
...
arangoimpsupportsWindows(CRLF)andUnix(LF)linebreaks.Linebreaksmightalsooccurinsidevaluesthatareenclosedwiththequotecharacter.
ThedefaultseparatorforCSVfilesisthecomma.Itcanbechangedusingthe--separatorparameterwheninvokingarangoimp.Thequotecharacterdefaultstothedoublequote(").Tousealiteraldoublequoteinsidea"column"intheimportdata,usetwodoublequotes.Tochangethequotecharacter,usethe--quoteparameter.Touseabackslashforescapingquotecharacters,pleasesettheoption--backslash-escapetotrue.
Changingthedatabaseandserverendpoint
Bydefault,arangoimpwillconnecttothedefaultdatabaseon127.0.0.1:8529withausernamedroot.Tochangethis,usethefollowingparameters:
server.database:nameofthedatabasetousewhenimporting(default:_system)server.endpoint:addressoftheArangoDBserver(default:tcp://127.0.0.1:8529)
Usingauthentication
arangoimpwillbydefaultsendanusernamerootandanemptypasswordtotheArangoDBserver.ThisisArangoDB'sdefaultconfiguration,anditshouldbechanged.Tomakearangoimpuseadifferentusernameorpassword,thefollowingcommand-lineargumentscanbeused:
server.username:username,usedifauthenticationisenabledonserverserver.password:passwordforuser,usedifauthenticationisenabledonserver
Thepasswordargumentcanalsobeomittedinordertoavoidhavingitsavedintheshell'scommand-linehistory.Whenspecifyingausernamebutomittingthepasswordparameter,arangoimpwillpromptforapassword.
Additionalparameters
Bydefault,arangoimpwillimportdataintothespecifiedcollectionbutwillnottouchexistingdata.Oftenitisconvenienttofirstremovealldatafromacollectionandthenruntheimport.arangoimpsupportsthiswiththeoptional--overwriteflag.Whensettingittotrue,alldocumentsinthecollectionwillberemovedpriortotheimport.
Author:JanSteemann
Tags:#arangoimp#import
ImportingData
88
Replicatingdatafromdifferentdatabases
Problem
Youhavetwoormoredifferentdatabaseswithvariousdatarespectivelycollectionsineachoneofthis,butyouwantyourdatatobecollectedatoneplace.
Note:ForthissolutionyouneedatleastArango2.0andyoumustrunthescriptineverydatabaseyouwanttobecollectdatafrom.
SolutionFirstofallyouhavetostartaserveronendpoint:
arangod--server.endpointtcp://127.0.0.1:8529
NowyouhavetocreatetwocollectionsandnamethemdataandreplicationStatus
db._create("data");
db._create("replicationStatus");
Savethefollowingscriptinafilenamedjs/common/modules/org/mysync.js
varinternal=require("internal");
//maximumnumberofchangesthatwecanhandle
varmaxChanges=1000;
//URLofcentralnode
vartransferUrl="http://127.0.0.1:8599/_api/import?collection=central&type=auto&createCollection=true&complete=true";
vartransferOptions={
method:"POST",
timeout:60
};
//thecollectionthatkeepsthestatusofwhatgotreplicatedtocentralnode
varreplicationCollection=internal.db.replicationStatus;
//thecollectioncontainingalldatachanges
varchangesCollection=internal.db.data;
functionkeyCompare(l,r){
if(l.length!=r.length){
returnl.length-r.length<0?-1:1;
}
//lengthisequal
for(i=0;i<l.length;++i){
if(l[i]!=r[i]){
returnl[i]<r[i]?-1:1;
}
}
return0;
};
functionlogger(msg){
"usestrict";
require("console").log("%s",msg);
}
functionreplicate(){
"usestrict";
ReplicatingData
89
varkey="status";//const
varstatus,newStatus;
try{
//fetchthepreviousreplicationstate
status=replicationCollection.document(key);
newStatus={_key:key,lastKey:status.lastKey};
}
catch(err){
//nopreviousreplicationstate.startfromthebeginning
newStatus={_key:key,lastKey:"0"};
}
//fetchthelatestchanges(needtoreversethembecause`last`returnsnewestchangesfirst)
varchanges=changesCollection.last(maxChanges).reverse(),change;
vartransfer=[];
for(changeinchanges){
if(changes.hasOwnProperty(change)){
vardoc=changes[change];
if(keyCompare(doc._key,newStatus.lastKey)<=0){
//alreadyhandledinapreviousreplicationrun
continue;
}
//documentsweneedtotransfer
//ifnecessary,wecouldrewritethedocumentshere,e.g.insert
//extravalues,createclient-specifickeysetc.
transfer.push(doc);
if(keyCompare(doc._key,newStatus.lastKey)>0){
//keeptrackofhighestkey
newStatus.lastKey=doc._key;
}
}
}
if(transfer.length===0){
//nothingtodo
logger("nothingtotransfer");
return;
}
logger("transferring"+transfer.length+"document(s)");
//nowtransferthedocumentstotheremoteserver
varresult=internal.download(transferUrl,JSON.stringify(transfer),transferOptions);
if(result.code>=200&&result.code<=202){
logger("centralserveracceptedthedocuments:"+JSON.stringify(result));
}
else{
//error
logger("centralserverdidnotacceptthedocuments:"+JSON.stringify(result));
throw"replicationerror";
}
//updatethereplicationstate
if(status){
//needtoupdatethepreviousreplicationstate
replicationCollection.update(key,newStatus);
}
else{
//needtoinsertthereplicationstate(1sttime)
replicationCollection.save(newStatus);
}
logger("deletingolddocuments");
//finallyremoveallelementsthatwetransferredsuccessfullyfromthechangescollection
//noneedtokeepthem
transfer.forEach(function(k){
changesCollection.remove(k);
});
}
exports.execute=function(param){
ReplicatingData
90
"usestrict";
logger("replicationwakeup");
replicate();
logger("replicationshutdown");
};
AfterwardschangetheURLofthecentralnodeinthescripttotheoneyouchosenbefore-e.g.tcp://127.0.0.1:8599
Nowregisterthescriptasarecurringaction:
require("internal").definePeriodic(1,10,"org/arangodb/mysync","execute","");
Note:Atthispointyoucanchangethetimethescriptwillbeexecuted.
Comment
Theserverstartedonendpointwillbethecentralnode.Itcollectschangesfromthelocalnodebyreplicatingitsdata.Thescriptwillpickupeverythingthathasbeenchangedsincethelastalterationinyourdatacollection.Every10seconds-orthetimeyouchosen-thescriptwillbeexecutedandsendthechangeddatatothecentralnodewhereitwillbeimportedintoacollectionnamedcentral.Afterthatthetransferreddatawillberemovedfromthedatacollection.
Ifyouwanttotestyourscriptsimplyaddsomedatatoyourdatacollection-e.g.:
for(i=0;i<100;++i)db.data.save({value:i});
Author:JanSteemann
Tags:#database#collection
ReplicatingData
91
XCopyinstallArangoDBonWindows
Problem
Evenifthereisaniceguidedinstallerforwindowsusers,notalluserspreferthiskindofinstallation.InordertohaveaportableapplicationXCOPYdeploymentisnecessary.
Solution
AsofVersion2.5.1ArangoDBdoesn'trelyonregistryentriesanymoresowecandeployusingaZIP-file.
Steps
Unziparchive
Openanexplorer,chooseaplacewhereyouwantArangoDBtobeandunzipthefilesthere.Itwillcreateitsowntopleveldirectorywiththeversionnumberinthestring.
Alterconfiguration
Optional:
Editetc\arangodb3\arangod.confifthedefaultvaluesdon'tsuityourneedslike:
thelocationofthedatabasefilesportstobind
andsoon.
CreateRuntimedirectories
arangodleansontheexistenceofsomedirectoriesinthevarsubdirectory,soyoushouldcreatethem:
C:\ProgramFiles\ArangoDB-3.1.11>mkdirvar\lib\arangodb
C:\ProgramFiles\ArangoDB-3.1.11>mkdirvar\lib\arangodb-apps
Runarangod
Tostartthedatabasesimplyrunit:
C:\ProgramFiles\ArangoDB-3.1.11>usr\bin\arangod
Nowittakesawhiletoopenallitsdatabases,loadsystemfacilities,bootstraptheJavaScriptenvironmentsandmanymore.Onceit'sreadytheoutputis:
INFOArangoDB(version3.1.11[windows])isreadyforbusiness.Havefun!
Nowyoucanopentheadministrativewebinterfaceinyourbrowserusinghttp://127.0.0.1:8529/.
Installingasservice
Ifyoudon'twanttorunarangodfromacmd-shelleachtimeinstallingitasasystemserviceistherightthingtodo.Thisrequiresadministrativeprivileges.YouneedtoRunasAdministratorthecmd-shell.FirstweneedtogranttheSYSTEM-useraccesstoourdatabasedirectory,sincearangodisgoingtoberunningasthatuser:
XCopyInstallWindows
92
C:\ProgramFiles\ArangoDB-3.1.11>icaclsvar/grantSYSTEM:F/t
Nextwecaninstalltheserviceitself:
C:\ProgramFiles\ArangoDB-3.1.11>usr\bin\arangod--install-service
NowyouwillhaveanewentryintheServicesdialoglabeledArangoDB-themulti-purposedatabase.Youcanstartitthereorjustdoitonthecommandlineusing:
C:\ProgramFiles\ArangoDB-3.1.11>NETSTARTArangoDB
Itwilltakeasimilaramountoftimetostartfromthecomandlineabovetilltheserviceisupandrunning.Sinceyoudon'thaveanyconsoletoinspectthestartup,messagesoftheseverityFATAL&ERRORarealsooutputintothewindowseventlog,soincaseoffailureyoucanhavealookattheEventlogintheManagementconsole
Author:WilfriedGoesgens
Tags:#windows#install
XCopyInstallWindows
93
MigrationfromArangoDB2.8to3.0
Problem
IwanttouseArangoDB3.0fromnowonbutIstillhavedatainArangoDB2.8.Ineedtomigratemydata.IamrunninganArangoDB3.0cluster(andpossiblyaclusterwithArangoDB2.8aswell).
Solution
TheinternaldataformatchangedcompletelyfromArangoDB2.8to3.0,thereforeyouhavetodumpalldatausingarangodumpandthenrestoreittothenewArangoDBinstanceusingarangorestore.
Generalinstructionsforthisprocedurecanbefoundinthemanual.Here,wecoversomeadditionaldetailsabouttheclustercase.
DumpingthedatainArangoDB2.8
Basically,dumpingthedataworkswiththefollowingcommand(usearangodumpfromyourArangoDB2.8distribution!):
arangodump--server.endpointtcp://localhost:8530--output-directorydump
oravariationofit,fordetailsseetheabovementionedmanualpageandthissection.IfyourArangoDB2.8instanceisacluster,simplyuseoneofthecoordinatorendpointsastheabove--server.endpoint.
RestoringthedatainArangoDB3.0
TheoutputconsistsofJSONfilesintheoutputdirectory,twoforeachcollection,oneforthestructureandoneforthedata.Thedataformatis100%compatiblewithArangoDB3.0,exceptthatArangoDB3.0hasanadditionaloptioninthestructurefilesforsynchronousreplication,namelytheattributereplicationFactor,whichisusedtospecify,howmanycopiesofthedataforeachshardarekeptinthecluster.
Therefore,youcansimplyusethiscommand(usethearangorestorefromyourArangoDB3.0distribution!):
arangorestore--server.endpointtcp://localhost:8530--input-directorydump
toimportyourdataintoyournewArangoDB3.0instance.Seethispagefordetailsontheavailablecommandlineoptions.IfyourArangoDB3.0instanceisacluster,thensimplyuseoneofthecoordinatorsas--server.endpoint.
Thatisit,yourdataismigrated.
Controlingthenumberofshardsandthereplicationfactor
Thisprocedureworksforallfourcombinationsofsingleserverandclusterforsourceanddestinationrespectively.Ifthetargetisasingleserverallsimplyworks.
Soitremainstoexplainhowonecontrolsthenumberofshardsandthereplicationfactorifthedestinationisacluster.
Ifthesourcewasacluster,arangorestorewillusethesamenumberofshardsasbefore,ifyoudonottellitotherwise.SinceArangoDB2.8doesnothavesynchronousreplication,itdoesnotproducedumpswiththereplicationFactorattribute,andsoarangorestorewillusereplicationfactor1forallcollections.Ifthesourcewasasingleserver,thesamewillhappen,additionally,arangorestorewillalwayscreatecollectionswithjustasingleshard.
Thereareessentially3waystochangethisbehaviour:
1. ThefirstistocreatethecollectionsexplicitlyontheArangoDB3.0cluster,andthensetthe--create-collectionfalseflag.Inthiscaseyoucancontrolthenumberofshardsandthereplicationfactorforeachcollectionindividuallywhenyoucreatethem.
2. Thesecondistousearangorestore'soptions--default-number-of-shardsand--default-replication-factor(thisoptionwas
Migrating2.8to3.0
94
introducedinVersion3.0.2)respectivelytospecifydefaultvalues,whicharetakenifthedumpfilesdonotspecifynumbers.Thismeansthatallsuchrestoredcollectionswillhavethesamenumberofshardsandreplicationfactor.
3. Ifyouneedmorecontrolyoucansimplyeditthestructurefilesinthedump.TheyaresimplyJSONfiles,youcanevenfirstuseaJSONprettyprintertomakeeditingeasier.ForthereplicationfactoryousimplyhavetoaddareplicationFactorattributetotheparameterssubobjectwithanumericalvalue.Forthenumberofshards,locatetheshardssubattributeoftheparametersattributeandeditit,suchthatithastherightnumberofattributes.Theactualnamesoftheattributesaswellastheirvaluesdonotmatter.Alternatively,addanumberOfShardsattributetotheparameterssubobject,thiswilloverridetheshardsattribute(thispossibilitywasintroducedinVersion3.0.2).
Notethatyoucanremoveindividualcollectionsfromyourdumpbydeletingtheirpairofstructureanddatafileinthedumpdirectory.Inthiswayyoucanrestoreyourdatainseveralstepsorevenparallelisetherestoreoperationbyrunningmultiplearangorestoreprocessesconcurrentlyondifferentdumpdirectories.Youshouldconsiderusingdifferentcoordinatorsforthedifferentarangorestoreprocessesinthiscase.
AllthesepossibilitiestogethergiveyoufullcontrolovertheshardinglayoutofyourdatainthenewArangoDB3.0cluster.
Migrating2.8to3.0
95
Showgrantsfunction
Problem
I'mlookingforuserdatabasegrants
Solution
Createaglobalfunctioninyour.arangosh.rcfilelikethis:
global.show_grants=function(){
letstmt;
stmt=db._createStatement({"query":"FORuin_usersRETURN{\"user\":u.user,\"databases\":u.databases}"});
console.log(stmt.execute().toString());
};
Nowwhenyouenterinarangosh,youcancallshow_grants()function.
Functionoutexample
[objectArangoQueryCursor,count:3,hasMore:false]
[
{
"user":"foo",
"databases":{
"_system":"rw",
"bar":"rw"
}
},
{
"user":"foo2",
"databases":{
"bar":"rw"
}
},
{
"user":"root",
"databases":{
"*":"rw"
}
}
]
Showgrantsfunction
96
CompilingArangoDB
Problem
YouwanttomodifysourcesoraddyourownchangestoArangoDB.
Solution
Arangodb,asmanyotheropensourceprojectsnowadaysisstandingontheshoulderofgiants.Thisgivesusasolidfoundationtobringyouauniqfeatureset,butitintroducesalotofdependenciesthatneedtobeinplaceinordertocompilearangodb.
SincebuildinfrastructuresareverydifferentdependingonthetargetOS,chooseyourtargetfromtherecepiesbelow.
CompileonDebian
CompileonWindows
RunningCustomBuild
Recompilingjemalloc
OpenSSL1.1
Compiling/Build
97
CompilingonDebian
Problem
Youwanttocompileandrunthedevelbranch,forexampletotestabugfix.InthisexamplethesystemisDebianbased.
Solution
ThissolutionwasmadeusingafreshDebianTestingmachineonAmazonEC2.Forcompleteness,thestepspertainingtoAWSarealsoincludedinthisrecipe.
LaunchtheVM
Optional
LogintoyourAWSaccountandlaunchaninstanceofDebianTesting.Iusedan'm3.xlarge'sincethathasabunchofcores,morethanenoughmemory,optimizednetworkandtheinstancestoreisonSSDswhichcanbeswitchedtoprovisionedIOPs.
TheCurrentAMIID'scanbefoundintheDebianWiki:https://wiki.debian.org/Cloud/AmazonEC2Image/Jessie
Upgradetotheverylatestversion
Optional
OnceyourEC2instanceisup,loginadadminandsudosutobecomeroot.
First,weremovethebackportsandchangetheprimarysources.list
rm-rf/etc/apt/sources.list.d
echo"debhttp://http.debian.net/debiantestingmaincontrib">/etc/apt/sources.list
echo"deb-srchttp://http.debian.net/debiantestingmaincontrib">>/etc/apt/sources.list
Updateandupgradethesystem.Makesureyoudon'thaveanybroken/unconfiguredpackages.Sometimesyouneedtorunsafe/fullupgrademorethanonce.Whenyou'redone,reboot.
apt-getinstallaptitude
aptitude-yupdate
aptitude-ysafe-upgrade
aptitude-yfull-upgrade
reboot
Installbuilddependencies
Mandatory
BeforeyoucanbuildArangoDB,youneedafewpackagespre-installedonyoursystem.
Loginagainandinstallthem.
sudoaptitude-yinstallgit-core\
build-essential\
libssl-dev\
libjemalloc-dev\
cmake\
python2.7\
sudoaptitude-yinstalllibldap2-dev#Enterpriseversiononly
DownloadtheSource
CompileonDebian
98
Downloadthelatestsourceusinggit:
unix>gitclonegit://github.com/arangodb/arangodb.git
Thiswillautomaticallyclonethedevelbranch.
Note:ifyouonlyplantocompileArangoDBlocallyanddonotwanttomodifyorpushanychanges,youcanspeedupcloningsubstantiallybyusingthe--single-branchand--depthparametersfortheclonecommandasfollows:
unix>gitclone--single-branch--depth1git://github.com/arangodb/arangodb.git
Setup
SwitchintotheArangoDBdirectory
unix>cdarangodb
unix>mkdirbuild
unix>cdbuild
Inordertogeneratethebuildenvironmentpleaseexecute
unix>cmake..
tosetuptheMakefiles.Thiswillcheckthevarioussystemcharacteristicsandinstalledlibraries.Ifyouinstalledthecompilerinanonstandardlocation,youmayneedtospecifyit:
cmake-DCMAKE_C_COMPILER=/opt/bin/gcc-DCMAKE_CXX_COMPILER=/opt/bin/g++..
IfyoucompileonMacOS,youshouldaddthefollowingoptionstothecmakecommand:
cmake..-DOPENSSL_ROOT_DIR=/usr/local/opt/openssl-DCMAKE_OSX_DEPLOYMENT_TARGET=10.11
IfyoualsoplantomakechangestothesourcecodeofArangoDB,youmaywanttocompilewiththeDebugbuildtype:
cmake..-DCMAKE_BUILD_TYPE=Debug
TheDebugtargetenablesadditionalsanitychecksetc.whichwouldslowdownproductionbinaries.Ifnobuildtypeisspecified,ArangoDBwillbecompiledwithbuildtypeRelWithDebInfo,whichisacompromisebetweengoodperformanceandmediumdebuggingexperience.
Otheroptionsvaluablefordevelopment:
-DUSE_MAINTAINER_MODE=On
NeededifyouplantomakechangestoAQLlanguage(whichisimplementedusingalexerandparserfilesinarangod/Aql/grammar.yandarangod/Aql/tokens.ll)orifyouwanttoenableruntimeassertions.Tousethemaintainermode,yoursystemhastocontainthetoolsFLEXandBISON.
-DUSE_BACKTRACE=On
UsethisoptionifyouwanttohaveC++stacktracesattachedtoyourexceptions.Thiscanbeusefultomorequicklylocatetheplacewhereanexceptionoranassertionwasthrown.Notethatthisoptionwillslowdowntheproducesbinariesabitandrequiresbuildingwithmaintainermode.
-DUSE_OPTIMIZE_FOR_ARCHITECTURE=On
CompileonDebian
99
Thiswilloptimizethebinaryforthetargetarchitecture,potentiallyenablingmorecompileroptimizations,butmakingtheresultingbinarylessportable.
ArangoDBwillthenautomaticallyusetheconfigurationfromfileetc/relative/arangod.conf.
-DUSE_FAILURE_TESTS=On
Thisoptionactivatesadditionalcodeintheserverthatintentionallymakestheservercrashormisbehave(e.g.bypretendingthesystemranoutofmemory)whencertaintestsarerun.Thisoptionisusefulforwritingtests.
-DUSE_JEMALLOC=Off
BydefaultArangoDBwillbebuiltwithabundledversionoftheJEMallocallocator.ThishoweverwillnotworkwhenusingruntimeanalyzerssuchasASANorValgrind.InordertousethesetoolsforinstrumentinganArangoDBbinary,JEMallocmustbeturnedoffduringcompilation.
sharedmemory
GypisusedasmakefilegeneratorbyV8.Gyprequiressharedmemorytobeavailable,whichmaynotifyoui.e.compileinachroot.Youcanmakeitavailablelikethis:
none/opt/chroots/ubuntu_precise_x64/dev/shmtmpfsrw,nosuid,nodev,noexec02
devpts/opt/chroots/ubuntu_precise_x64/dev/ptsdevptsgid=5,mode=62000
Compilation
Compiletheprograms(server,client,utilities)byexecuting
make
inthebuildsubdirectory.ThiswillcompileArangoDBandcreatethebinaryexecutableinfilebuild/bin/arangod.
Startingandtesting
Checkthebinarybystartingitusingthecommandline.
unix>build/bin/arangod-cetc/relative/arangod.conf--server.endpointtcp://127.0.0.1:8529/tmp/database-dir
ThiswillstartuptheArangoDBandlistenforHTTPrequestsonport8529boundtoIPaddress127.0.0.1.Youshouldseethestartupmessagessimilartothefollowing:
2016-06-01T12:47:29Z[29266]INFOArangoDBxxx...
2016-06-10T12:47:29Z[29266]INFOusingendpoint'tcp://127.0.0.1.8529'fornon-encryptedrequests
2016-06-01T12:47:30Z[29266]INFOAuthenticationisturnedon
2016-60-01T12:47:30Z[29266]INFOArangoDB(versionxxx)isreadyforbusiness.Havefun!
Ifitfailswithamessageaboutthedatabasedirectory,pleasemakesurethedatabasedirectoryyouspecifiedexistsandcanbewritteninto.
UseyourfavoritebrowsertoaccesstheURL
http://127.0.0.1:8529/
ThisshouldbringupArangoDB'swebinterface.
Re-buildingArangoDBafteranupdate
Tostayup-to-datewithchangesmadeinthemainArangoDBrepository,youwillneedtopullthechangesfromitandre-runmake.
CompileonDebian
100
Normally,thiswillbeassimpleasfollows:
unix>gitpull
unix>(cdbuild&&make)
FromtimetotimetherewillbebiggerstructuralchangesinArangoDB,whichmayrendertheoldMakefilesinvalid.Shouldthisbethecaseandmakecomplainsaboutmissingfilesetc.,thefollowingcommandsshouldfixit:
unix>rm-rfbuild/*
unix>cdbuild&&cmake..<cmakeoptionsgohere>
unix>(cdbuild&&make)
NotethattheabovecommandswillrunafullrebuildofArangoDBandallofitsthird-partycomponents.Thatwilltakeawhiletocomplete.
Installation
InalocaldevelopmentenvironmentitisnotnecessarytoinstallArangoDBsomewhere,becauseitcanbestartedfromwithinthesourcedirectoryasshownabove.
IfthereshouldbetheneedtoinstallArangoDB,executethefollowingcommand:
(cdbuild&&sudomakeinstall)
Theserverwillbydefaultbeinstalledin
/usr/local/sbin/arangod
Theconfigurationfilewillbeinstalledin
/usr/local/etc/arangodb/arangod.conf
Thedatabasewillbeinstalledin
/usr/local/var/lib/arangodb
TheArangoShellwillbeinstalledin
/usr/local/bin/arangosh
Youshouldaddanarangodbuserandgroup(asroot),plusmakesureitownsthesedirectories:
useradd-garangodbarangodb
chown-Rarangodb:arangodb/usr/local/var/lib/arangodb3-apps/
chown-Rarangodb:arangodb/tmp/database-dir/
Note:Theinstallationdirectorywillbedifferentifyouuseoneoftheprecompiledpackages.Pleasecheckthedefaultlocationsofyouroperatingsystem,e.g./etcand/var/lib.
WhenupgradingfromapreviousversionofArangoDB,pleasemakesureyouinspectArangoDB'slogfileafteranupgrade.ItmayalsobenecessarytostartArangoDBwiththe--database.auto-upgradeparameteroncetoperformrequiredupgradeorinitializationtasks.
Author:PatrickHuberAuthor:WilfriedGoesgens
Tags:#debian#driver
CompileonDebian
101
CompilingArangoDBunderWindows
Problem
IwanttocompileArangoDB3.0andonwardsunderWindows.
Note:ForthisrecipeyouneedatleastArangoDB3.0;ForArangoDBversionbefore3.0lookattheoldCompilingArangoDBunderWindows.
SolutionWithArangoDB3.0acompletecmakeenvironmentwasintroduced.Thisalsostreamlinesthedependenciesonwindows.Wesugesttousechocolatey.orgtoinstallmostofthedependencies.Forsuremostprojectsoffertheirownsetup&installpackages,chocolateyoffersasimplifiedwaytoinstallthemwithlessuserinteractions.Youcanevenusechocolateyviathebrandnewansibles2.0winrmfacilitytodounattendedinstallionsofsomesoftwareonwindows-thecoolthinglinuxguysalwaystoldyouabout.
Ingredients
Firstinstallthechocopackagemanagerbypastingthistinycmdletintoacommandwindow(needstoberunwithAdministratorprivileges;Rightclickstartmenu,CommandPrompt(Admin)):
@powershell-NoProfile-ExecutionPolicyBypass-Command"iex((new-objectnet.webclient).DownloadString('https://chocolatey.org
/install.ps1'))"&&SETPATH=%PATH%;%ALLUSERSPROFILE%\chocolatey\bin
VisualStudioanditsCompiler
SincechococurrentlyfailstoaltertheenvironmentforMicrosoftVisualStudio,wesuggesttodownloadandinstallVisualStudiobyhand.CurrentlyVisualStudio2015istheonlysupportedoption.
Youneedtomakesurethatitinstallstheoption"ProgrammingLanguages/C++",elsecmakewillfailtodectectitlateron.
Afteritsuccessfullyinstalled,startitonce,soitcanfinishitssetup.
Moredependencies
Nowyoucaninvokethechocopackagemanagerforanunattendedinstallofthedependencies(needstoberunwithAdministratorprivilegesagain):
chocoinstall-ycmake.portablensispython2procdumpwindbgwgetnuget.commandline
ThenwefetchtheOpenSSLlibraryviathenugetcommandlineclient(doesn'tneedAdministratorprivileges):
nugetinstallopenssl
Optional
Ifyouintendtoruntheunittestsorcompilefromgit,youalsoneed(needstoberunwithAdministratorprivilegesagain):
chocoinstall-ygitwinflexbisonruby
CloseandreopentheAdministratorcommandwindowinordertocontinuewiththerubydevkit:
chocoinstall-yruby2.devkit
CompileonWindows
102
AndmanuallyinstalltherequirementsviatheGemfilefetchedfromtheArangoDBGitrepository(needstoberunwithAdministratorprivileges):
wgethttps://raw.githubusercontent.com/arangodb/arangodb/devel/UnitTests/HttpInterface/Gemfile
setPATH=%PATH%;C:\tools\DevKit2\bin;C:\tools\DevKit2\mingw\bin
geminstallbundler
bundler
NotethattheV8buildscriptsandgyparen'tcompatiblewithPython3.xhenceyouneedpython2!
BuildingArangoDB
Downloadandextractthereleasetarballfromhttps://www.arangodb.com/download/
Orclonethegithubrepository,andcheckoutthebranchortagyouneed(devel,3.0)
gitclonehttps://github.com/arangodb/arangodb.git-bdevel
cdarangodb
GeneratetheVisualstudioprojectfiles,andcheckbackthatcmakediscoveredallcomponentsonyoursystem:
mkdirBuild64
cdBuild64
cmake-G"VisualStudio14Win64"..
Notethatinsomecasescmakestrugglestofindtheproperpythoninterpreter(i.e.thecygwinonewon'twork).Youcanforceoverruleitbyappending:
-DPYTHON_EXECUTABLE:FILEPATH=C:/tools/python2/python.exe
YoucannowloadtheseintheVisualStudioIDEorusecmaketostartthebuild:
cmake--build.--configRelWithDebInfo
ThebinariesneedtheICUdatafileicudt54l.dat,whichisautomaticallycopiedintothedirectorycontainingtheexecutable.
Fordevelopment,unittestsanddocumentation:Cygwin(Optional)Thedocumentationandunittestsstillrequireacygwinenvironment.Herethehintshowtogetitproperlyinstalled:
Youneedatleastmakefromcygwin.Cygwinalsooffersacmake.Donotinstallthecygwincmake.
Youshouldalsoissuethesecommandstogenerateuserinformationsforthecygwincommands:
mkpasswd>/etc/passwd
mkgroup>/etc/group
TurningACLoff(noacl)forallmountsincygwinfixespermissionstroublesthatmayappearinthebuild:
#/etc/fstab
#
#ThisfileisreadoncebythefirstprocessinaCygwinprocesstree.
#Topickupchanges,restartallCygwinprocesses.Foradescription
#seehttps://cygwin.com/cygwin-ug-net/using.html#mount-table
#noacl=IgnoreAccessControlListandletWindowshandlepermissions
C:/cygwin64/bin/usr/binntfsbinary,auto,noacl00
C:/cygwin64/lib/usr/libntfsbinary,auto,noacl00
C:/cygwin64/ntfsoverride,binary,auto,noacl00
CompileonWindows
103
none/cygdrivecygdrivebinary,posix=0,user,noacl00
EnablenativesymlinksforCygwinandgitCygwinwillcreateproprietaryfilesasplaceholdersbydefaultinsteadofactuallysymlinkingfiles.TheplaceholderslatertellCygwinwheretoresolvepathsto.Itdoesnotintercepteveryaccesstotheplaceholdershowever,sothat3rdpartyscriptsbreak.WindowsVistaandabovesupportrealsymlinks,andCygwincanbeconfiguredtomakeuseofit:
#useactualsymlinkstopreventdocumentationbuilderrors
#(requireselevatedrights!)
exportCYGWIN="winsymlinks:native"
NotethatyoumustrunCygwinasadministratororchangetheWindowsgrouppoliciestoallowuseraccountstocreatesymlinks(gpedit.mscifavailable).
BTW:YoucancreatesymlinksmanuallyonWindowslike:
mklink/Htarget/file.extsource/file.ext
mklink/Dtarget/pathsource/path
mklink/Jtarget/pathsource/path/for/junction
AndinCygwin:
ln-ssourcetarget
MakingtheICUdatabasepublicallyavailableIfyouintendtousethemachinefordevelopmentpurposes,itmaybemorepracticaltocopyittoacommonplace:
cp3rdParty/V8/V8-5.0.71.39/third_party/icu/source/data/in/icudtl.dat/cygdrive/c/Windows/icudt54l.dat
Andconfigureyourenvironment(yesthisinstructionrememberstothehitchhikersguidetothegalaxy...)sothatICU_DATApointstoc:\\Windows.Youdothatbyopeningtheexplorer,rightclickonThisPCinthetreeontheleft,choosePropertiesintheopeningwindowAdvancedsystemsettings,inthePopupEnvironmentVariables,anotherpopupopens,intheSystemVariablespartyouclickNew,Andvariablename:ICU_DATAtothevalue:c:\\Windows
CompileonWindows
104
RunningUnitests(Optional)
Youcanthenruntheunittestsinthecygwinshelllikethat:
build64/bin/RelWithDebInfo/arangosh.exe\
-cetc/relative/arangosh.conf\
--log.levelwarning\
--server.endpointtcp://127.0.0.1:1024\
--javascript.executeUnitTests/unittest.js\
--\
all\
--rubyc:/tools/ruby22/bin/ruby\
--rspecc:/tools/ruby22/bin/rspec\
--buildTypeRelWithDebInfo\
--skipNondeterministictrue\
--skipTimeCriticaltrue
--skipBoosttrue\
--skipGeotrue
Documentation(Optional)NodeJS(needstoberunwithAdministratorprivilegesagain):
chocoinstall-ynodejs
Gitbook:
npminstall-ggitbook-cli
CompileonWindows
105
Markdown-pp:
gitclonehttps://github.com/triAGENS/markdown-pp.git
cdmarkdown-pp
pythonsetup.pyinstall
Ditaa:
Downloadandinstall:http://ditaa.sourceforge.net/#download
Authors:FrankCeller,WilfriedGoesgensandSimranBrucherseifer.
Tags:#windows
CompileonWindows
106
OpenSSLOpenSSL1.1isonitswaytomainstream.Sofar(ArangoDB3.2)hasonlybeenthorouglytestedwithOpenSSL1.0and1.1isunsupported.
Buildingagainst1.1willcurrentlyresultinacompileerror:
/arangodb/arangodb/lib/SimpleHttpClient/SslClientConnection.cpp:224:14:error:useofundeclaredidentifier'SSLv2_method'
meth=SSLv2_method();
^
/arangodb/arangodb/lib/SimpleHttpClient/SslClientConnection.cpp:239:14:warning:'TLSv1_method'isdeprecated[-Wdeprecated-dec
larations]
meth=TLSv1_method();
^
/usr/include/openssl/ssl.h:1612:45:note:'TLSv1_method'hasbeenexplicitlymarkeddeprecatedhere
DEPRECATEDIN_1_1_0(__owurconstSSL_METHOD*TLSv1_method(void))/*TLSv1.0*/
^
/arangodb/arangodb/lib/SimpleHttpClient/SslClientConnection.cpp:243:14:warning:'TLSv1_2_method'isdeprecated[-Wdeprecated-d
eclarations]
meth=TLSv1_2_method();
Youshouldinstallopenssl1.0(shouldbepossibletoinstallitalongside1.1).
Afterthathelpcmaketofindthe1.0variant.
ExampleonArchLinux:
cmake-DOPENSSL_INCLUDE_DIR=/usr/include/openssl-1.0/-DOPENSSL_SSL_LIBRARY=/usr/lib/libssl.so.1.0.0-DOPENSSL_CRYPTO_LIBRARY=/
usr/lib/libcrypto.so.1.0.0<SOURCE_PATH>
AfterthatArangoDBshouldcompilefine.
OpenSSL
107
Runningacustombuild
Problem
You'vealreadybuiltacustomversionofArangoDBandwanttorunit.Possiblyinisolationfromanexistinginstallationoryoumaywanttore-usethedata.
Solution
First,youneedtobuildyourownversionofArangoDB.Ifyouhaven'tdonesoalready,havealookatanyoftheCompilingrecipes.
Thisrecipeassumesyou'reintherootdirectoryoftheArangoDBdistributionandcompilinghassuccessfullyfinished.
Runninginisolation
Thispartshowshowtorunyourcustombuildwithanemptydatabasedirectory
#createdatadirectory
mkdir/tmp/arangodb
#run
bin/arangod\
--configurationetc/relative/arangod.conf\
--database.directory/tmp/arangodb
Runningwithdata
Thispartshowshowtorunyourcustombuildwiththeconfiganddatafromapre-existingstableinstallation.
BEWAREArangoDB'sdevelopersmaychangethedbfileformatandafterrunningwithachangedfileformat,theremaybenowayback.Alternativelyyoucanrunyourbuildinisolationanddumpandrestorethedatafromthestabletoyourcustombuild.
Whenrunninglikethis,youmustrunthedbasthearangoduser(thedefaultinstalledbythepackage)inordertohavewriteaccesstothelog,databasedirectoryetc.Runningasrootwilllikelymessupthefilepermissions-goodluckfixingthat!
#becomerootfirst
su
#nowswitchtoarangodandrun
su-arangod
bin/arangod--configuration/etc/arangodb/arangod.conf
Author:PatrickHuber
Tags:#build
RunningCustomBuild
108
JemallocThisarticleisonlyrelevantifyouintendtocompilearangodbonUbuntu16.10ordebiantesting
Onmoremodernlinuxsystems(development/floatingatthetimeofthiswriting)youmaygetcompile/linkerrorswitharangodbregardingjemalloc.ThisisduetocompilersswitchingtheirdefaultbehaviourregardingthePIC-PositionIndependendCode.Itseemscommonthatjemallocremainsinastagewherethischangeisn'tfollowedandcausesarangodbtoerroroutduringthelinkingphase.
Fromnowoncmakewilldetectthisandgiveyouthishint:
thestaticsystemjemallocisn'tsuitable!Recompilewiththecurrentcompilerordisableusing`-DCMAKE_CXX_FLAGS=-no-pie-DCM
AKE_C_FLAGS=-no-pie`
Nowyou'vegotthreechoices.
Doingwithoutjemalloc
Fixesthecompilationissue,butyouwillgetproblemswiththeglibcsheapfragmentationbehaviourwhichinthelongerrunwillleadtoaneverincreasingmemoryconsumptionofArangoDB.
So,whilethismaybesuitablefordevelopment/testingsystems,itsdefinitelynotforproduction.
DisablingPICaltogetherThiswillbuildanarangodwhichdoesn'tusethiscompilerfeature.Itmaybenotsonicefordevelopmentbuilds.Itcanbeachievedbyspecifyingtheseoptionsoncmake:
-DCMAKE_CXX_FLAGS=-no-pie-DCMAKE_C_FLAGS=-no-pie
Recompilejemalloc
Thesmartestwayistofixthejemalloclibrariespackagesonyoursystemsoitsreflectingthatnewbehaviour.Ondebian/ubuntusystemsitcanbeachievedlikethis:
apt-getinstallautomakedebhelperdocbook-xslxsltprocdpkg-dev
aptsourcejemalloc
cdjemalloc*
dpkg-buildpackage
cd..
dpkg-i*jemalloc*deb
Recompilingjemalloc
109
Cloud,DCOSandDocker
AmazonWebServices(AWS)
RunningonAWS
UpdateonAWS
MicrosoftAzureRunningonAzure
DockerDockerArangoDB
DockerwithNodeJSApp
GiantSwarm
IntheGiantSwarm
Mesos/DCOS
ArangoDBinMesos
DC/OS:Fullexample
Cloud,DCOSandDocker
110
RunningArangoDBonAWSArangoDBisavailableasAMIontheAWSMarketplace.
(Ifyou'vealreadyarunningArangoDBimageonAWSandneedanupdate,pleasehavealookatUpdatingArangoDBonAWS).
Hereisaquickguidehowtostart:
GotheArangoDBmarketplace,selectthelatestversionandclickonContinueUsethe1-ClickLaunchtabandselectthesizeoftheinstance(EC2InstanceType)youwishtouse.NowyoucancontinuewithaclickonAcceptTerms&Launchwith1-Click.
Note:Ifyoudonothaveakeypairawarningwillappearafterclickingandyouwillbeaskedtogenerateakeypair.
YousuccessfullylaunchedanArangoDBinstanceonAWS.
TheArangoDBWeb-InterfacecanbereachedusingtheAccessSoftwarebuttonorviapublicinstanceIPandthePort8529(e.g.:http://12.13.14.15:8529)ThedefaultuserisrootandthepasswordistheInstanceID(YoucanfindtheInstanceIDontheinstancelist).
RunningonAWS
111
IfyouwanttolearnmoreaboutArangoDB,startwiththe[ArangoDBFirstSteps][../../Manual/GettingStarted/index.html]inourDocumentationortryoneofourTutorialsorCookbookrecipes.
Author:IngoFriepoertner
Tags:#aws,#amazon,#howto
RunningonAWS
112
UpdatinganArangoDBImageonAWSIfyourunanArangoDBonAWSandusedtheprovidedAMIintheAWSMarketplace,youatsomepointwanttoupdatetothelatestrelease.TheprocesstosubmitandpublishanewArangoDBimagetothemarketplacetakessometimeandyoumightnotfindthelatestreleaseinthemarketplacestoreyet.
However,updatingtothelatestversionisnotthathard.
First,logintothevirtualmachinewiththeuserubuntuandthepublicDNSnameoftheinstance.
TostartanupdatetoaknownversionofArangoDByoucanuse:
sudoapt-getupdate
sudoapt-getinstallarangodb=2.5.7
ToupgradeanArangoDBinstancetoanewmajorversion(from2.5.xto2.6.x),use:
sudoapt-getinstallarangodb
Youmightgetawarningthattheconfigurationfilehaschanged:
Configurationfile'/etc/arangodb/arangod.conf'
==>Modified(byyouorbyascript)sinceinstallation.
==>Packagedistributorhasshippedanupdatedversion.
Whatwouldyouliketodoaboutit?Youroptionsare:
YorI:installthepackagemaintainer'sversion
NorO:keepyourcurrently-installedversion
D:showthedifferencesbetweentheversions
Z:startashelltoexaminethesituation
Thedefaultactionistokeepyourcurrentversion.
***arangod.conf(Y/I/N/O/D/Z)[default=N]?
Youshouldstaywiththecurrentconfiguration(type"N"),astherearesomechangesmadeintheconfigurationforAWS.Ifyoutype"Y"youwillloseaccessfromyourapplicationstothedatabasesomakesurethatdatabasedirectoryandserverendpointarevalid.
--server.database-directory
needstobe`/vol/...`forAWS
--server.endpoint
needstobe`tcp://0.0.0.0:8529`forAWS
Ifyouupdatetoanewmajorversion,youwillbeaskedtoupgradesothatadatabasemigrationcanbestarted:
sudoservicearangodbupgrade
sudoservicearangodbstart
NowArangoDBshouldbebacktonormal.
Fornowwehavetostickwiththismanualprocessbutemightcreateasimplerupdateprocessinthefuture.PleaseprovidefeedbackhowyouuseourAmazonAMIandhowwecanimproveyouruserexperience.
Author:IngoFriepoertner
Tags:#aws#upgrade
UpdateonAWS
113
ArangoDBinMicrosoftAzureIwanttouseArangoDBinMicrosoftAzure
Howto
Theshortansweris:goto
https://vmdepot.msopentech.com/
typein"ArangoDB",selecttheversionyourequireandpress"CreateVirtualMachine".
FollowtheinstructionsgiventhereandwithinminutesyouhavearunningArangoDBinstanceinMicrosoftAzure.Youwillreceiveanemailassoonasyourmachineisready.
Assumeyourmachineiscalledmyarangodb,thanyoucanaccessArangoDBpointingyourbrowserto
http://myarangodb.cloudapp.net:8529
Pleasenotethatforsecurityreasonsthedefaultinstanceispasswordprotected.
However,thepasswordfor"root"isempty.So,pleaseloginandchangethepasswordassoonaspossible.
Authors:FrankCeller
Tags:#azure,#howto
RunningonAzure
114
HowtorunArangoDBinaDockercontainer
Problem
HowdoyoumakeArangoDBruninaDockercontainer?
Solution
ArangoDBisnowavailableasanofficialrepositoryintheDockerHub(@seedocumentationthere).
Author:FrankCeller
Tags:#docker#howto
DockerArangoDB
115
ArangoDB,NodeJSandDocker
Problem
I'mlookingforaheadstartinusingtheArangoDBdockerimage.
Solution
WewillusetheguessergameforArangoDBfrom
https://github.com/arangodb/guesser
Thisisasimplegameguessinganimalsorthings.ItlearnswhileplayingandstoresthelearnedinformationinanArangoDBinstance.Thegameiswrittenusingtheexpressframework.
Note:Youneedtoswitchtothedockerbranch.
Thegamehasthetwocomponents
front-endwithnode.jsandexpressback-endwithArangoDBandFoxx
Thereforetheguessergameneedstwodockercontainers,onecontainerforthenode.jsservertorunthefront-endcodeandonecontainerforArangoDBforthestorageback-end.
NodeServer
ThegameisitselfcanbeinstallviaNPMorfromgithub.Thereisanimageavailablefromdockerhubcalledarangodb/example-guesserwhichisbasedontheDockerfilefromgithub.
Youcaneitherbuildthedockercontainerlocallyorsimplyusetheavailableonefromdockerhub.
unix>dockerrun-p8000:8000-enolink=1arangodb/example-guesser
Startingwithoutadatabaselink
UsingDB-Serverhttp://localhost:8529
Guesserappserverlisteningathttp://0.0.0.0:8000
Thiswillstart-upnodeandtheguessergameisavailableonport8000.Nowpointyourbrowsertoport8000.Youshouldseethestart-upscreen.However,withoutastoragebackenditwillbeprettyuseless.Therefore,stopthecontainerandproceedwiththenextstep.
Ifyouwanttobuildthecontainerlocally,checkouttheguessergamefrom
https://github.com/arangodb/example-guesser
Switchintothedocker/nodesubdirectoryandexecutedockerbuild..
ArangoDB
ArangoDBisalreadyavailableondocker,sowestartaninstance
unix>dockerrun--namearangodb-guesserarangodb/arangodb
showalloptions:
dockerrun-ehelp=1arangodb
startingArangoDBinstand-alonemode
DockerwithNodeJSApp
116
That'sit.Notethatinanproductiveenvironmentyouwouldneedtoattachastoragecontainertoit.Weignorethishereforthesakeofsimplicity.
GuesserGame
SomeTesting
UsetheguessergameimagetostarttheArangoDBshellandlinktheArangoDBinstancetoit.
unix>dockerrun--linkarangodb-guesser:db-link-itarangodb/example-guesserarangosh--server.endpoint@DB_LINK_PORT_8529_TCP
@
Theparameter--linkarangodb-guesser:db-linklinkstherunningArangoDBintotheapplicationcontainerandsetsanenvironmentvariableDB_LINK_PORT_8529_TCPwhichpointstotheexposedportoftheArangoDBcontainer:
DB_LINK_PORT_8529_TCP=tcp://172.17.0.17:8529
YourIPmayvary.Thecommandarangosh...attheendofdockercommandexecutestheArangoDBshellinsteadofthedefaultnodecommand.
Welcometoarangosh2.3.1[linux].Copyright(c)ArangoDBGmbH
UsingGoogleV83.16.14JavaScriptengine,READLINE6.3,ICU52.1
Prettyprintingvalues.
ConnectedtoArangoDB'tcp://172.17.0.17:8529'version:2.3.1,database:'_system',username:'root'
Type'tutorial'foratutorialor'help'toseecommonexamples
arangosh[_system]>
Theimportantlineis
ConnectedtoArangoDB'tcp://172.17.0.17:8529'version:2.3.1,database:'_system',username:'root'
Ittellsyouthattheapplicationcontainerwasabletoconnecttothedatabaseback-end.PressControl-Dtoexit.
StartUpTheGame
Readytoplay?Startthefront-endcontainerwiththedatabaselinkandinitializethedatabase.
unix>dockerrun--linkarangodb-guesser:db-link-p8000:8000-einit=1arangodb/example-guesser
Useyourbrowsertoplaythegameattheaddresshttp://127.0.0.1:8000/.The
-einit=1
isonlyneedthefirsttimeyoustart-upthefront-endandonlyonce.Thenexttimeyourunthefront-endorifyoustartasecondfront-endserveruse
unix>dockerrun--linkarangodb-guesser:db-link-p8000:8000arangodb/example-guesser
Author:FrankCeller
Tags:#docker
DockerwithNodeJSApp
117
ArangoDBintheGiantSwarmusingDockercontainers
Problem
IwanttouseArangoDBintheGiantSwarmwithDockercontainers.
Solution
GiantSwarmallowsyoutodescribeanddeployyourapplicationbyprovidingasimpleJSONdescription.Thecurrentweatherappisagoodexampleonhowtoinstallanapplicationwhichusestwocomponents,namelynodeandredis.
MycolleagueMaxhaswrittenaguessergamewithvariousfront-endsandArangoDBasbackend.InordertogetthefeelingofbeingpartoftheGiantSwarm,Ihavestartedtosetupthisgameintheswarm.
FirstSteps
Theguessergameconsistsofafront-endwrittenasexpressapplicationinnodeandastorageback-endusingArangoDBandasmallAPIdevelopedwithFoxx.
Thefront-endapplicationisavailableasimage
arangodb/example-guesser
andtheArangoDBback-endwiththeFoxxAPIas
arangodb/example-guesser-db
Thedockerfilesusedtocreatetheimagesareavailablefromgithub
https://github.com/arangodb/guesser
SetuptheSwarm
Setupyourswarmenvironmentasdescribedinthedocumentation.Createaconfigurationfilefortheswarmcalledarangodb.jsonandfireuptheapplication
{
"app_name":"guesser",
"services":[
{
"service_name":"guesser-game",
"components":[
{
"component_name":"guesser-front-end",
"image":"arangodb/example-guesser",
"ports":[8000],
"dependencies":[
{"name":"guesser-back-end","port":8529}
],
"domains":{"guesser.gigantic.io":8000}
},
{
"component_name":"guesser-back-end",
"image":"arangodb/example-guesser-db",
"ports":[8529]
}
]
}
]
}
IntheGiantSwarm
118
Thisdefinesanapplicationguesserwithasingleserviceguesser-game.Thisservicehastwocomponentsguesser-front-endandguesser-back-end.Thedockerimagesaredownloadedfromthestandarddockerrepository.
Theline
"domains":{"guesser.gigantic.io":8000}
exposestheinternalport8000totheexternalportonport80forthehostguesser.gigantic.io.
InordertotellGiantSwarmaboutyourapplication,execute
unix>swarmcreatearangodb.json
Creating'arangodb'inthe'fceller/dev'environment...
Appcreatedsuccessfully!
Thiswillcreateanapplicationcalledguesser.
unix>swarmstatusguesser
Appguesserisdown
servicecomponentinstanceidstatus
guesser-gameguesser-back-end5347e718-3d27-4356-b530-b24fc5d1e3f5down
guesser-gameguesser-front-end7cf25b43-13c4-4dd3-9a2b-a1e32c43ae0ddown
Weseethetwocomponentsofourapplication.Botharecurrentlypowereddown.
StartuptheGuesserGame
Startingyourenginesisnowonesimplecommand
unix>swarmstartguesser
Startingapplicationguesser...
Applicationguesserisup
Nowtheapplicationisup
unix>swarmstatusguesser
Appguesserisup
servicecomponentinstanceidstatus
guesser-gameguesser-back-end5347e718-3d27-4356-b530-b24fc5d1e3f5up
guesser-gameguesser-front-end7cf25b43-13c4-4dd3-9a2b-a1e32c43ae0dup
Pointyourbrowserto
http://guesser.gigantic.io
andguessananimal.
Ifyouwanttocheckthelogfilesofaninstanceyoucanasktheswarmgivingittheinstanceid.Forexample,theback-end
unix>swarmlogs5347e718-3d27-4356-b530-b24fc5d1e3f5
2014-12-1712:34:57.984554+0000UTC-systemd-StoppingUserguesser-back-end...
2014-12-1712:36:28.074673+0000UTC-systemd-5cfe11d6-343e-49bb-8029-06333844401f.servicestop-sigtermtimedout.Killing.
2014-12-1712:36:28.077821+0000UTC-systemd-5cfe11d6-343e-49bb-8029-06333844401f.service:mainprocessexited,code=killed
,status=9/KILL
2014-12-1712:36:38.213245+0000UTC-systemd-StoppedUserguesser-back-end.
2014-12-1712:36:38.213543+0000UTC-systemd-Unit5cfe11d6-343e-49bb-8029-06333844401f.serviceenteredfailedstate.
2014-12-1712:37:55.074158+0000UTC-systemd-StartingUserguesser-back-end...
2014-12-1712:37:55.208354+0000UTC-docker-Pullingrepositoryarangodb/example-guesser-db
2014-12-1712:37:56.995122+0000UTC-docker-Status:Imageisuptodateforarangodb/example-guesser-db:latest
2014-12-1712:37:57.000922+0000UTC-systemd-StartedUserguesser-back-end.
2014-12-1712:37:57.707575+0000UTC-docker--->startingArangoDB
IntheGiantSwarm
119
2014-12-1712:37:57.708182+0000UTC-docker--->waitingforArangoDBtobecomeready
2014-12-1712:38:28.157338+0000UTC-docker--->installingguessergame
2014-12-1712:38:28.59025+0000UTC-docker--->readyforbusiness
andthefront-end
unix>swarmlogs7cf25b43-13c4-4dd3-9a2b-a1e32c43ae0d
2014-12-1712:35:10.139684+0000UTC-systemd-StoppingUserguesser-front-end...
2014-12-1712:36:40.32462+0000UTC-systemd-aa7756a4-7a87-4633-bea3-e416d035188b.servicestop-sigtermtimedout.Killing.
2014-12-1712:36:40.327754+0000UTC-systemd-aa7756a4-7a87-4633-bea3-e416d035188b.service:mainprocessexited,code=killed
,status=9/KILL
2014-12-1712:36:50.567911+0000UTC-systemd-StoppedUserguesser-front-end.
2014-12-1712:36:50.568204+0000UTC-systemd-Unitaa7756a4-7a87-4633-bea3-e416d035188b.serviceenteredfailedstate.
2014-12-1712:38:04.796129+0000UTC-systemd-StartingUserguesser-front-end...
2014-12-1712:38:04.921273+0000UTC-docker-Pullingrepositoryarangodb/example-guesser
2014-12-1712:38:06.459366+0000UTC-docker-Status:Imageisuptodateforarangodb/example-guesser:latest
2014-12-1712:38:06.469988+0000UTC-systemd-StartedUserguesser-front-end.
2014-12-1712:38:07.391149+0000UTC-docker-UsingDB-Serverhttp://172.17.0.183:8529
2014-12-1712:38:07.613982+0000UTC-docker-Guesserappserverlisteningathttp://0.0.0.0:8000
ScalingUp
Yourgamebecomesasuccess.Well,scalingupthefront-endistrivial.
Simplychangeyourconfigurationfileandrecreatetheapplication:
{
"app_name":"guesser",
"services":[
{
"service_name":"guesser-game",
"components":[
{
"component_name":"guesser-front-end",
"image":"arangodb/example-guesser",
"ports":[8000],
"dependencies":[
{"name":"guesser-back-end","port":8529}
],
"domains":{"guesser.gigantic.io":8000},
"scaling_policy":{"min":2,"max":2}
},
{
"component_name":"guesser-back-end",
"image":"arangodb/example-guesser-db",
"ports":[8529]
}
]
}
]
}
Theimportantlineis
"scaling_policy":{"min":2,"max":2}
Ittellstheswarmtousetwofront-endcontainers.Inlaterversionoftheswarmyouwillbeabletochangethenumberofcontainersinarunningapplicationwiththecommand:
>swarmscaleupguesser/guesser-game/guesser-front-end--count=1
Scalingupcomponentguesser/guesser-game/guesser-front-endby1...
WeatArangoDBarehardatworktomakescalinguptheback-enddatabaseequallyeasy.Staytunedfornewreleasesinearly2015...
Authors:FrankCeller
Tags:#docker,#giantswarm,#howto
IntheGiantSwarm
120
IntheGiantSwarm
121
ArangoDBonApacheMesosusingMarathonandDocker
Problem
IwanttouseArangoDBinApacheMesoswithDockercontainers.
Solution
MesosinitsnewestversionmakesitveryeasytouseArangoDB,becauseMesoshasaddedsupportfordockercontainers.TogetherwithMarathontostartthefront-endandback-endpartsofanapplication,installationisstraightforward.
MycolleagueMaxhaswrittenaguessergamewithvariousfront-endsandArangoDBasbackend.InordertogetthefeelingofbeingpartoftheMesosphere,IhavestartedtosetupthisgameinanDigitalOceanenvironment.
FirstSteps
Theguessergameconsistsofafront-endwrittenasexpressapplicationinnodeandastorageback-endusingArangoDBandasmallAPIdevelopedwiththeFoxxmicroservicesframework.
Thefront-endapplicationisavailableasimage
arangodb/example-guesser
andtheArangoDBback-endwiththeFoxxAPIas
arangodb/example-guesser-db
Thedockerfilesusedtocreatetheimagesareavailablefromgithub
https://github.com/arangodb/guesser
SetUptheEnvironment
FollowtheinstructionsonMesospheretosetupanenvironmentwithdockersupport.YoushouldendupwithsshaccesstotheMesosmaster.
SetUptheApplication
ForthistutorialwebindthedatabasetoafixedportontheMesosenvironment.Pleasenote,thatthemesosphereusesHAproxytomaptheglobalporttotherealhostandport.TheserverscreatedbyMesospherewillhaveaHAproxydefinedonallmastersandslaves.
Thatmeans,ifwechose32333asserviceportforthedatabase,itwillbereachableonthisportonallmastersandslaves.Theappdefinitionforthedatabaselookslike
{
"id":"/guesser/database",
"apps":[
{
"id":"/guesser/database/arangodb",
"container":{
"docker":{
"image":"arangodb/example-guesser-db",
"network":"BRIDGE",
"portMappings":[
{"containerPort":8529,"hostPort":0,"servicePort":32222,"protocol":"tcp"}
]
}
},
ArangoDBinMesos
122
"cpus":0.2,
"mem":512.0,
"instances":1
}
]
}
Thiswillstartthedockerimagefortheback-endandbindstheportto32222.
InsidethedockercontaineranenvironmentvariableHOSTissetbethemesosslavetopointtotheslave.Thefront-endcanthereforeaccessport32222onthishosttocontacttheHAproxy,gainingaccesstothedatabase.
Theappdefinitionforthefront-endlookslike
{
"id":"/guesser/frontend",
"apps":[
{
"id":"/guesser/frontend/node",
"container":{
"docker":{
"image":"arangodb/example-guesser",
"network":"BRIDGE",
"portMappings":[
{"containerPort":8000,"hostPort":0,"servicePort":32221,"protocol":"tcp"}
]
}
},
"cpus":0.2,
"mem":256.0,
"instances":1
}
]
}
Marathonallowstodefineagroupofapplicationswithdependenciesbetweenthecomponents.Thefront-enddependsontheback-end,thereforethecompletegroupdefinitionslookslike
{
"id":"/guesser",
"groups":[
{
"id":"/guesser/database",
"apps":[
{
"id":"/guesser/database/arangodb",
"container":{
"docker":{
"image":"arangodb/example-guesser-db",
"network":"BRIDGE",
"portMappings":[
{"containerPort":8529,"hostPort":0,"servicePort":32222,"protocol":"tcp"}
]
}
},
"cpus":0.2,
"mem":512.0,
"instances":1
}
]
},
{
"id":"/guesser/frontend",
"dependencies":["/guesser/database"],
"apps":[
{
"id":"/guesser/frontend/node",
"container":{
"docker":{
"image":"arangodb/example-guesser",
"network":"BRIDGE",
"portMappings":[
ArangoDBinMesos
123
{"containerPort":8000,"hostPort":0,"servicePort":32221,"protocol":"tcp"}
]
}
},
"cpus":0.2,
"mem":256.0,
"instances":1
}
]
}
]
}
Thisstartsoneinstanceoftheback-endcalled/guesser/database/arangodbandoneinstanceofthefront-endcalled/guesser/frontend/node.Thefront-enddependsontheback-end.
Inordertofireuptheguessergamesavetheabovedefinitioninafileguesser.jsonandexecute
curl-XPUT-H"Accept:application/json"-H"Content-Type:application/json"127.0.0.1:8080/v2/groups-d"`catguesser.json`"
onthemesosmaster.
IfyounowswitchtotheMarathonconsoleonport8080,youshouldseeapps,namely/guesser/database/arangodband/guesser/frontend/node.
Ifyouaccessport32222,youshouldseetheArangoDBconsole.
Andfinally,onport32211,youcanplaytheguessergame.
ArangoDBinMesos
124
ScalingUp
Yourgamebecomesasuccess.Well,scalingupthefront-endistrivial.Simply,gotothemarathonpageandscaleup/guesser/frontend/node.
Authors:FrankCeller
Tags:#docker,#mesos,#mesosphere,#howto
ArangoDBinMesos
125
DeployingahighlyavailableapplicationusingArangoDBandFoxxonDC/OS
Problem
HowcanIdeployanapplicationusingArangoDBonDC/OSandmakeeverythinghighlyavailable?
Solution
Toachievethisgoalseveralindividualcomponentshavetobecombined.
InstallDC/OS
Gotohttps://dcos.io/install/andfollowtheinstructionstoinstallDC/OS
AlsomakesuretoinstallthedcosCLI.
InstallArangoDB
OnceyourclusterisDC/OSclusterisreadyinstallthepackagearangodb3fromtheuniversetab(defaultsettingsarefine)
DetailedinstructionsmaybefoundinthefirstchaptersofourDC/OStutorialhere:
https://dcos.io/docs/1.7/usage/tutorials/arangodb/
TounderstandhowArangoDBensuresthatitishighlyavailablemakesuretoreadtheclusterdocumentationhere:
ArangoDBArchitectureDocumentation
Deployaloadbalancerforthecoordinators
OnceArangoDBisinstalledandhealthyyoucanaccesstheclusterviaoneofthecoordinators.
TodosofromtheoutsideDC/OSprovidesaniceandsecuregatewaythroughtheiradmininterface.
Howeverthisisintendedtobeusedfromtheoutsideonly.ApplicationsusingArangoDBasitsdatastorewillwanttoconnecttothecoordinatorsfromtheinside.YoucouldcheckthetasklistwithinDC/OStofindouttheendpointswherethecoordinatorsarelistening.Howeverthesearenottobetrusted:Theycanfailatanytime,thenumberofcoordinatorsmightchangeduetoup-anddownscalingorsomeonemightevenkillafullDC/OSAgentandtasksmaysimplyfailandreappearonadifferentendpoint.
Inshort:Endpointsaretemporary.
Tomitigatethisproblemwehavebuiltaspecialloadbalancerforthesecoordinators.
Toinstallit:
$gitclonehttps://github.com/arangodb/arangodb-mesos-haproxy
$cdarangodb-mesos-haproxy
$dcosmarathonappaddmarathon.json
Afterwardsyoucanusethefollowingendpointtoaccessthecoordinatorsfromwithinthecluster:
tcp://arangodb-proxy.marathon.mesos:8529
Tomakeithighlyavailableyoucansimplylaunchafewmoreinstancesusingthemarathonwebinterface.DetailsonhowtodothisandhowtodeployanapplicationusingtheUIcanbefoundhere:https://dcos.io/docs/1.7/usage/tutorials/marathon/marathon101/
Ourtestapplication
DC/OS:Fullexample
126
NowthatwehavesetupArangoDBonDC/OSitistimetodeployourapplication.Inthisexamplewewilluseourguesserapplicationwhichyoucanfindhere:
https://github.com/arangodb/guesser
ThisapplicationhassomeapplicationcodeandaFoxxmicroservice.
DeployingtheFoxxservice
OpentheArangoDBinterface(viatheServicestabintheDC/OSinterface)andgotoServices.
Enter/guesserasmountdirectoryChoosegithubonthetabandenterthefollowingrepository:
ArangoDB/guesser
Choosemasterasversion.
PressInstall
Deploytheapplication
Finallyitistimetodeploytheapplicationcode.Wehavepackagedeverythingintoadockercontainer.Theonlythingthatismissingissomeconnectioninfoforthedatabase.Thiscanbeprovidedviaenvironmentvariablesthroughmarathon.
OpenthemarathonwebinterfaceinyourDC/OScluster(Servicesandthenmarathon).
ThenclickCreateapplication
OnthetoprightyoucanchangetotheJSONview.Pastethefollowingconfig:
{
"id":"/guesser",
"cmd":null,
"cpus":1,
"mem":128,
"disk":0,
"instances":3,
"container":{
"type":"DOCKER",
"volumes":[],
"docker":{
"image":"arangodb/example-guesser",
"network":"BRIDGE",
"portMappings":[
{
"containerPort":8000,
"hostPort":0,
"servicePort":10004,
"protocol":"tcp"
}
],
"privileged":false,
"parameters":[],
"forcePullImage":true
}
},
"labels":{
"HAPROXY_GROUP":"external"
},
"env":{
"ARANGODB_SERVER":"http://arangodb-proxy.marathon.mesos:8529",
"ARANGODB_ENDPOINT":"tcp://arangodb-proxy.marathon.mesos:8529"
}
}
AsyoucanseeweareprovidingtheARANGODB_ENDPOINTasanenvironmentvariable.Thedockercontainerwilltakethatanduseitwhenconnecting.Thisconfigurationinjectionviaenvironmentvariablesisconsideredadockerbestpracticeandthisishowyoushouldprobablycreateyourapplicationsaswell.
DC/OS:Fullexample
127
Nowwehaveourguesserappstartedwithinmesos.
Itishighlyavailablerightawayaswelaunched3instances.ToscaleitupordownsimplyusethescalebuttonsinthemarathonUI.
Makeitpublicallyavailable
Forthistoworkweneedanothertool,namelymarathon-lb
Installit:
dcospackageinstallmarathon-lb
Afterinstallationitwillscanallmarathonapplicationsforaspecialsetoflabelsandmaketheseapplicationsavailabletothepublic.
Tomaketheguesserapplicationavailabletothepublicyoufirsthavetodetermineahostnamethatpointstotheexternalloadbalancerinyourenvironment.WheninstallingusingthecloudformationtemplateonAWSthisisthehostnameofthesocalledpublicslave.Youcanfinditintheoutputtabofthecloudformationtemplate.
Inmycasethiswas:
mop-publicslaveloa-3phq11mb7oez-1979417947.eu-west-1.elb.amazonaws.com
Incasethereareuppercasedletterspleasetakeextracaretolowercasethemasthemarathon-lbwillfailotherwise.
EditthesettingsoftheguesserappinthemarathonUIandaddthishostnameasthelabelHAPROXY_0_VHOSTeitherusingtheUIorusingtheJSONmode:
[...]
"labels":{
"HAPROXY_GROUP":"external",
"HAPROXY_0_VHOST":"mop-publicslaveloa-3phq11mb7oez-1979417947.eu-west-1.elb.amazonaws.com"
},
[...]
Toscaleitupandthusmakingithighlyavailableincreasetheinstancescountwithinmarathon.
FormoredetailedinformationandmoreconfigurationoptionsincludingSSLetcbesuretocheckthedocumentation:
https://docs.mesosphere.com/1.7/usage/service-discovery/marathon-lb/usage/
Accessingtheguessergame
Aftermarathon-lbhasreloadeditsconfiguration(whichshouldhappenalmostimmediately)youshouldbeabletoaccesstheguessergamebypointingawebbrowsertoyourhostname.
Havefun!
Conclusion
Therearequiteafewcomponentsinvolvedinthisprocessbutonceyouarefinishedyouwillhaveahighlyresilientsystemwhichsurelywasworththeeffort.Noneofthecomponentsinvolvedisasinglepointoffailure.
Author:AndreasStreichardt
Tags:#docker#howto#dcos#cluster#ha
DC/OS:Fullexample
128
MonitoringArangoDBusingcollectd
Problem
TheArangoDBwebinterfaceshowsanicesummaryofthecurrentstate.IwanttoseesimilarnumbersinmymonitoringsystemsoIcananalyzethesystemusagepostmortemorsendalarmsonfailure.
Solution
CollectdisanexcellenttooltogatherallkindsofmetricsfromasystemanddeliverittoacentralmonitoringlikeGraphiteand/orNagios.
Ingredients
Forthisrecipeyouneedtoinstallthefollowingtools:
collectd>=5.4.2TheaggregationDaemonkcollectdforinspectingthedata
Configuringcollectd
ForaggregatingthevalueswewillusethecURL-JSONplug-in.WewillstorethevaluesusingtheRound-Robin-Databasewriter(RRD)whichkcollectdcanlateronpresenttoyou.
Weassumeyourcollectdcomesfromyourdistributionandreadsitsconfigfrom/etc/collectd/collectd.conf.Sincethisfiletendstobecomeprettyunreadablequickly,weusetheincludemechanism:
<Include"/etc/collectd/collectd.conf.d">
Filter"*.conf"
</Include>
Thiswaywecanmakeeachmetricgrouponcompactsetconfigfiles.Itconsistsofthreecomponents:
loadingtheplug-inaddingmetricstotheTypesDBtheconfigurationfortheplug-initself
rrdtool
WewillusetheRound-Robin-Databaseasstoragebackendfornow.Itcreatesitsowndatabasefilesoffixedsizeforeachspecifictimerange.Lateryoumaychoosemoreadvancedwriter-plug-ins,whichmaydonetworkdistributionofyourmetricsorintegratetheabovementionedGraphiteoryouralreadyestablishedmonitoring,etc.
FortheRRDwewillgoprettymuchwithdefaults:
#Loadtheplug-in:
LoadPluginrrdtool
<Pluginrrdtool>
DataDir"/var/lib/collectd/rrd"
#CacheTimeout120
#CacheFlush900
#WritesPerSecond30
#CreateFilesAsyncfalse
#RandomTimeout0
#
#Thefollowingsettingsareratheradvanced
#andshouldusuallynotbetouched:
#StepSize10
#HeartBeat20
Monitoring
129
#RRARows1200
#RRATimespan158112000
#XFF0.1
</Plugin>
cURLJSON
Collectdcomeswithawiderangeofmetricaggregationplug-ins.ManytoolstodayuseJSONasdataformatinggrammar;sodoesArangoDB.Thereforeaplug-inofferingtofetchJSONdocumentsviaHTTPistheperfectmatchasanintegrationinterface:
#Loadtheplug-in:
LoadPlugincurl_json
#weneedtouseourowntypestogenerateindividualnamesforourgauges:
TypesDB"/etc/collectd/collectd.conf.d/arangodb_types.db"
<Plugincurl_json>
#AdjusttheURLsocollectdcanreachyourarangod:
<URL"http://localhost:8529/_db/_system/_admin/aardvark/statistics/short">
#SetyourauthenticationtoAardvarkhere:
#User"foo"
#Password"bar"
<Key"totalTimeDistributionPercent/values/0">
Type"totalTimeDistributionPercent_values"
</Key>
<Key"totalTimeDistributionPercent/cuts/0">
Type"totalTimeDistributionPercent_cuts"
</Key>
<Key"requestTimeDistributionPercent/values/0">
Type"requestTimeDistributionPercent_values"
</Key>
<Key"requestTimeDistributionPercent/cuts/0">
Type"requestTimeDistributionPercent_cuts"
</Key>
<Key"queueTimeDistributionPercent/values/0">
Type"queueTimeDistributionPercent_values"
</Key>
<Key"queueTimeDistributionPercent/cuts/0">
Type"queueTimeDistributionPercent_cuts"
</Key>
<Key"bytesSentDistributionPercent/values/0">
Type"bytesSentDistributionPercent_values"
</Key>
<Key"bytesSentDistributionPercent/cuts/0">
Type"bytesSentDistributionPercent_cuts"
</Key>
<Key"bytesReceivedDistributionPercent/values/0">
Type"bytesReceivedDistributionPercent_values"
</Key>
<Key"bytesReceivedDistributionPercent/cuts/0">
Type"bytesReceivedDistributionPercent_cuts"
</Key>
<Key"numberOfThreadsCurrent">
Type"gauge"
</Key>
<Key"numberOfThreadsPercentChange">
Type"gauge"
</Key>
<Key"virtualSizeCurrent">
Type"gauge"
</Key>
<Key"virtualSizePercentChange">
Type"gauge"
</Key>
<Key"residentSizeCurrent">
Type"gauge"
</Key>
<Key"residentSizePercent">
Type"gauge"
</Key>
<Key"asyncPerSecondCurrent">
Type"gauge"
</Key>
<Key"asyncPerSecondPercentChange">
Type"gauge"
Monitoring
130
</Key>
<Key"syncPerSecondCurrent">
Type"gauge"
</Key>
<Key"syncPerSecondPercentChange">
Type"gauge"
</Key>
<Key"clientConnectionsCurrent">
Type"gauge"
</Key>
<Key"clientConnectionsPercentChange">
Type"gauge"
</Key>
<Key"physicalMemory">
Type"gauge"
</Key>
<Key"nextStart">
Type"gauge"
</Key>
<Key"waitFor">
Type"gauge"
</Key>
<Key"numberOfThreads15M">
Type"gauge"
</Key>
<Key"numberOfThreads15MPercentChange">
Type"gauge"
</Key>
<Key"virtualSize15M">
Type"gauge"
</Key>
<Key"virtualSize15MPercentChange">
Type"gauge"
</Key>
<Key"asyncPerSecond15M">
Type"gauge"
</Key>
<Key"asyncPerSecond15MPercentChange">
Type"gauge"
</Key>
<Key"syncPerSecond15M">
Type"gauge"
</Key>
<Key"syncPerSecond15MPercentChange">
Type"gauge"
</Key>
<Key"clientConnections15M">
Type"gauge"
</Key>
<Key"clientConnections15MPercentChange">
Type"gauge"
</Key>
</URL>
</Plugin>
Tocircumventtheshortcomingofthecurl_JSONplug-intoonlytakethelastpathelementasnameforthemetric,weneedtogivethemanameusingourowntypes.dbfilein/etc/collectd/collectd.conf.d/arangodb_types.db:
totalTimeDistributionPercent_valuesvalue:GAUGE:U:U
totalTimeDistributionPercent_cutsvalue:GAUGE:U:U
requestTimeDistributionPercent_valuesvalue:GAUGE:U:U
requestTimeDistributionPercent_cutsvalue:GAUGE:U:U
queueTimeDistributionPercent_valuesvalue:GAUGE:U:U
queueTimeDistributionPercent_cutsvalue:GAUGE:U:U
bytesSentDistributionPercent_valuesvalue:GAUGE:U:U
bytesSentDistributionPercent_cutsvalue:GAUGE:U:U
bytesReceivedDistributionPercent_valuesvalue:GAUGE:U:U
bytesReceivedDistributionPercent_cutsvalue:GAUGE:U:U
Rollingyourown
YoumaywanttomonitoryourownmetricsfromArangoDB.Hereisasimpleexamplehowtousetheconfig:
Monitoring
131
{
"testArray":[1,2],
"testArrayInbetween":[{"blarg":3},{"blub":4}],
"testDirectHit":5,
"testSubLevelHit":{"oneMoreLevel":6}
}
ThisconfigsnippetwillparsetheJSONabove:
<Key"testArray/0">
Type"gauge"
#Expect:1
</Key>
<Key"testArray/1">
Type"gauge"
#Expect:2
</Key>
<Key"testArrayInbetween/0/blarg">
Type"gauge"
#Expect:3
</Key>
<Key"testArrayInbetween/1/blub">
Type"gauge"
#Expect:4
</Key>
<Key"testDirectHit">
Type"gauge"
#Expect:5
</Key>
<Key"testSubLevelHit/oneMoreLevel">
Type"gauge"
#Expect:6
</Key
Getitserved
Nowwewill(re)startcollectdsoitpicksupourconfiguration:
/etc/init.d/collectdstart
Wewillinspectthesyslogtorevalidatenothingwentwrong:
Mar313:59:52localhostcollectd[11276]:Startingstatisticscollectionandmonitoringdaemon:collectd.
Mar313:59:52localhostsystemd[1]:StartedLSB:managethestatisticscollectiondaemon.
Mar313:59:52localhostcollectd[11283]:Initializationcomplete,enteringread-loop.
Collectdaddsthehostnametothedirectoryaddress,sonowweshouldhavefileslikethese:
-rw-r--r--1rootroot154888Mar216:53/var/lib/collectd/rrd/localhost/curl_json-default/gauge-numberOfThreads15M.rrd
NowwestartkcollectdtoviewthevaluesintheRRDfile:
Monitoring
132
Sincewestartedputtingvaluesinjustnow,weneedtochoose'lasthour'andzoominalittlemoretoinspectthevalues.
Finishedwiththisdish,waitformoremetricstocomeinotherrecipes.
Author:WilfriedGoesgens
Tags:#json#monitoring
Monitoring
133
MonitoringreplicationslaveNote:thisrecipeisworkingwithArangoDB2.5,youneedacollectdcurl_jsonpluginwithcorrectbooleantypemapping.
Problem
Howtomonitortheslavestatususingthecollectdcurl_JSONplugin.
Solution
SincearangodbreportsthereplicationstatusinJSON,integratingitwiththecollectdcurl_JSONpluginshouldbeaneasyexercise.However,onlyveryrecentversionsofcollectdwillhandlebooleanflagscorrectly.
Ourtestmaster/slavesetuprunswiththethemasterlisteningontcp://127.0.0.1:8529andtheslave(whichwequery)listeningontcp://127.0.0.1:8530.TheyreplicateadabatasebythenametestDatabase.
Sincereplicationappliersareactiveperdatabaseandourexampledoesn'tusethedefault_system,weneedtospecifyitsnameintheURLlikethis:_db/testDatabase.
Weneedtoparseadocumentfromarequestlikethis:
curl--dump-http://localhost:8530/_db/testDatabase/_api/replication/applier-state
Ifthereplicationisnotrunningthedocumentwilllooklikethat:
{
"state":{
"running":false,
"lastAppliedContinuousTick":null,
"lastProcessedContinuousTick":null,
"lastAvailableContinuousTick":null,
"safeResumeTick":null,
"progress":{
"time":"2015-11-02T13:24:07Z",
"message":"appliershutdown",
"failedConnects":0
},
"totalRequests":1,
"totalFailedConnects":0,
"totalEvents":0,
"totalOperationsExcluded":0,
"lastError":{
"time":"2015-11-02T13:24:07Z",
"errorMessage":"nostarttick",
"errorNum":1413
},
"time":"2015-11-02T13:31:53Z"
},
"server":{
"version":"2.7.0",
"serverId":"175584498800385"
},
"endpoint":"tcp://127.0.0.1:8529",
"database":"testDatabase"
}
Arunningreplicationwillreturnsomethinglikethis:
{
"state":{
"running":true,
"lastAppliedContinuousTick":"1150610894145",
"lastProcessedContinuousTick":"1150610894145",
Collectd-ReplicationSlaves
134
"lastAvailableContinuousTick":"1151639153985",
"safeResumeTick":"1150610894145",
"progress":{
"time":"2015-11-02T13:49:56Z",
"message":"fetchingmasterlogfromtick1150610894145",
"failedConnects":0
},
"totalRequests":12,
"totalFailedConnects":0,
"totalEvents":2,
"totalOperationsExcluded":0,
"lastError":{
"errorNum":0
},
"time":"2015-11-02T13:49:57Z"
},
"server":{
"version":"2.7.0",
"serverId":"175584498800385"
},
"endpoint":"tcp://127.0.0.1:8529",
"database":"testDatabase"
}
Wecreateasimplecollectdconfigurationin/etc/collectd/collectd.conf.d/slave_testDatabase.confthatmatchesourAPI:
TypesDB"/etc/collectd/collectd.conf.d/slavestate_types.db"
<Plugincurl_json>
#AdjusttheURLsocollectdcanreachyourarangodslaveinstance:
<URL"http://localhost:8530/_db/testDatabase/_api/replication/applier-state">
#Setyourauthenticationtothatdatabasehere:
#User"foo"
#Password"bar"
<Key"state/running">
Type"boolean"
</Key>
<Key"state/totalOperationsExcluded">
Type"counter"
</Key>
<Key"state/totalRequests">
Type"counter"
</Key>
<Key"state/totalFailedConnects">
Type"counter"
</Key>
</URL>
</Plugin>
Togetnicemetricnames,wespecifyourowntypes.dbfilein/etc/collectd/collectd.conf.d/slavestate_types.db:
booleanvalue:ABSOLUTE:0:1
So,basicallystate/runningwillgiveyou0/1ifits(not/)runningthroughthecollectdmonitor.
Author:WilfriedGoesgens
Tags:#monitoring#foxx#json
Collectd-ReplicationSlaves
135
MonitoringArangoDBClusternetworkusage
Problem
Werunaclusterandwanttoknowwhetherthetrafficisunbalancedorsomethinglikethat.Wewantacheapestimatewhichhosthashowmuchtraffic.
Solution
AswealreadyrunCollectdasourmetric-hub,wewanttoutilizeittoalsogiveusthesefigures.AverycheapwaytogeneratethesevaluesarethecountersintheIPTablesfirewallofoursystem.
Ingredients
Forthisrecipeyouneedtoinstallthefollowingtools:
collectd:theaggregationDaemonkcollectdforinspectingthedataiptables-shouldcomewithyourLinuxdistributionfermforcompactfirewallcodewebaseonMonitoringwithCollecdrecipeforunderstandingthebasicsaboutcollectd
GettingthestateandthePortsofyourcluster
Nowweneedtofindoutthecurrentconfigurationofourcluster.Forthetimebeingweassumeyousimplyissued
./scripts/startLocalCluster.sh
togetyousetup.Soyouknowyou'vegottwoDB-Servers-oneCoordinator,oneagent:
ps-eaf|greparango
arangod214061116:59pts/1400:00:00bin/etcd-arango--data-dir/var/tmp/tmp-21550-1347489353/shell_server/agentar
ango4001--nameagentarango4001--bind-addr127.0.0.1:4001--addr127.0.0.1:4001--peer-bind-addr127.0.0.1:7001--peer-addr12
7.0.0.1:7001--initial-cluster-statenew--initial-clusteragentarango4001=http://127.0.0.1:7001
arangod214081416:56pts/1400:00:01bin/arangod--database.directorycluster/data8629--cluster.agency-endpointt
cp://localhost:4001--cluster.my-addresstcp://localhost:8629--server.endpointtcp://localhost:8629--cluster.my-local-infodb
server:localhost:8629--log.filecluster/8629.log--cluster.my-idPavel
arangod214101516:56pts/1400:00:02bin/arangod--database.directorycluster/data8630--cluster.agency-endpointt
cp://localhost:4001--cluster.my-addresstcp://localhost:8630--server.endpointtcp://localhost:8630--cluster.my-local-infodb
server:localhost:8630--log.filecluster/8630.log--cluster.my-idPerry
arangod214161516:56pts/1400:00:02bin/arangod--database.directorycluster/data8530--cluster.agency-endpointt
cp://localhost:4001--cluster.my-addresstcp://localhost:8530--server.endpointtcp://localhost:8530--cluster.my-local-infoco
ordinator:localhost:8530--log.filecluster/8530.log--cluster.my-idClaus
Wecannowcheckwhichportstheyoccupied:
netstat-aplnt|greparango
tcp00127.0.0.1:70010.0.0.0:*LISTEN21406/etcd-arango
tcp00127.0.0.1:40010.0.0.0:*LISTEN21406/etcd-arango
tcp00127.0.0.1:85300.0.0.0:*LISTEN21416/arangod
tcp00127.0.0.1:86290.0.0.0:*LISTEN21408/arangod
tcp00127.0.0.1:86300.0.0.0:*LISTEN21410/arangod
Theagenthas7001and4001.Sinceit'srunninginsingleservermodeitsclusterport(7001)shouldnotshowanytraffic,port4001istheinterestingone.Claus-Thisisthecoordinator.YourApplicationwilltalktoitonport8530Pavel-ThisisthefirstDB-Server;Clauswilltalktoitonport8629Perry-ThisisthesecondDB-Server;Clauswilltalktoitonport8630
Collectd-Networkusage
136
ConfiguringIPTables/ferm
SincetheusualsolutionusingshellscriptscallingiptablesbringstheDRYprincipletoagrindinghold,weneedsomethingbetter.Herefermcomestotherescue-Itenablesyoutoproduceverycompactandwellreadablefirewallconfigurations.
Accordingtotheportswefoundinthelastsection,wewillconfigureourfirewallin/etc/ferm/ferm.conf,andputtheidentitiesintothecommentssowehaveapersistentnamingscheme:
#blindlyforwardthesetotheaccountingchain:
@def$ARANGO_RANGE=4000:9000;
@def&TCP_ACCOUNTING($PORT,$COMMENT,$SRCCHAIN)={
@def$FULLCOMMENT=@cat($COMMENT,"_",$SRCCHAIN);
dport$PORTmodcommentcomment$FULLCOMMENTNOP;
}
@def&ARANGO_ACCOUNTING($CHAINNAME)={
#Thecoordinators:
&TCP_ACCOUNTING(8530,"Claus",$CHAINNAME);
#Thedb-servers:
&TCP_ACCOUNTING(8629,"Pavel",$CHAINNAME);
&TCP_ACCOUNTING(8630,"Perry",$CHAINNAME);
#Theagency:
&TCP_ACCOUNTING(4001,"etcd_client",$CHAINNAME);
#itshouldn'ttalktoitselfifitisonlyrunningwithasingleinstance:
&TCP_ACCOUNTING(7007,"etcd_cluster",$CHAINNAME);
}
tablefilter{
chainINPUT{
prototcpdport$ARANGO_RANGE@subchain"Accounting"{
&ARANGO_ACCOUNTING("input");
}
policyDROP;
#connectiontracking
modstatestateINVALIDDROP;
modstatestate(ESTABLISHEDRELATED)ACCEPT;
#allowlocalpacket
interfaceloACCEPT;
#respondtoping
protoicmpACCEPT;
#allowIPsec
protoudpdport500ACCEPT;
proto(espah)ACCEPT;
#allowSSHconnections
prototcpdportsshACCEPT;
}
chainOUTPUT{
policyACCEPT;
prototcpdport$ARANGO_RANGE@subchain"Accounting"{
&ARANGO_ACCOUNTING("output");
}
#connectiontracking
#modstatestateINVALIDDROP;
modstatestate(ESTABLISHEDRELATED)ACCEPT;
}
chainFORWARD{
policyDROP;
#connectiontracking
modstatestateINVALIDDROP;
modstatestate(ESTABLISHEDRELATED)ACCEPT;
}
}
Note:Thisisaverybasicconfiguration,mainlywiththepurposetodemonstratetheaccountingfeature-sodon'trunthisinproduction)
Collectd-Networkusage
137
Afteractivatingitinteractivelywith
ferm-i/etc/ferm/ferm.conf
Wenowusetheiptablescommandlineutilitydirectlytoreviewthestatusourcurrentsetting:
iptables-L-nvx
ChainINPUT(policyDROP85packets,6046bytes)
pktsbytestargetprotoptinoutsourcedestination
76361821798Accountingtcp--**0.0.0.0/00.0.0.0/0tcpdpts:4000:9000
00DROPall--**0.0.0.0/00.0.0.0/0stateINVALID
1470014857709ACCEPTall--**0.0.0.0/00.0.0.0/0stateRELATED,ESTABLISHED
1307800ACCEPTall--lo*0.0.0.0/00.0.0.0/0
00ACCEPTicmp--**0.0.0.0/00.0.0.0/0
00ACCEPTudp--**0.0.0.0/00.0.0.0/0udpdpt:500
00ACCEPTesp--**0.0.0.0/00.0.0.0/0
00ACCEPTah--**0.0.0.0/00.0.0.0/0
00ACCEPTtcp--**0.0.0.0/00.0.0.0/0tcpdpt:22
ChainFORWARD(policyDROP0packets,0bytes)
pktsbytestargetprotoptinoutsourcedestination
00DROPall--**0.0.0.0/00.0.0.0/0stateINVALID
00ACCEPTall--**0.0.0.0/00.0.0.0/0stateRELATED,ESTABLISHED
ChainOUTPUT(policyACCEPT296packets,19404bytes)
pktsbytestargetprotoptinoutsourcedestination
77201882404Accountingtcp--**0.0.0.0/00.0.0.0/0tcpdpts:4000:9000
1457514884356ACCEPTall--**0.0.0.0/00.0.0.0/0stateRELATED,ESTABLISHED
ChainAccounting(2references)
pktsbytestargetprotoptinoutsourcedestination
20457750tcp--**0.0.0.0/00.0.0.0/0tcpdpt:8530/*Claus_input*/
2017890tcp--**0.0.0.0/00.0.0.0/0tcpdpt:8629/*Pavel_input*/
26297352tcp--**0.0.0.0/00.0.0.0/0tcpdpt:8630/*Perry_input*/
2604336184tcp--**0.0.0.0/00.0.0.0/0tcpdpt:4001/*etcd_client_inpu
t*/
00tcp--**0.0.0.0/00.0.0.0/0tcpdpt:7007/*etcd_cluster_inp
ut*/
20457750tcp--**0.0.0.0/00.0.0.0/0tcpdpt:8530/*Claus_output*/
2017890tcp--**0.0.0.0/00.0.0.0/0tcpdpt:8629/*Pavel_output*/
26297352tcp--**0.0.0.0/00.0.0.0/0tcpdpt:8630/*Perry_output*/
2604336184tcp--**0.0.0.0/00.0.0.0/0tcpdpt:4001/*etcd_client_outp
ut*/
00tcp--**0.0.0.0/00.0.0.0/0tcpdpt:7007/*etcd_cluster_out
put*/
YoucanseenicelytheAccountingsub-chainwithourcomments.Theseshouldbeprettystraightforwardtomatch.Wealsoseethepktsandbytescolumns.Theycontainthecurrentvalueofthesecountersofyoursystem.
Readmoreaboutlinuxfirewallingandfermconfigurationtobesureyoudotherightthing.
ConfiguringCollectdtopickupthesevalues
Sinceyoursystemnowgeneratesthesenumbers,wewanttoconfigurecollectdwithitsiptablesplugintoaggregatethem.
Wedosointhe/etc/collectd/collectd.conf.d/iptables.conf:
LoadPluginiptables
<Pluginiptables>
Chainfilter"Accounting""Claus_input"
Chainfilter"Accounting""Pavel_input"
Chainfilter"Accounting""Perry_input"
Chainfilter"Accounting""etcd_client_input"
Chainfilter"Accounting""etcd_cluster_input"
Chainfilter"Accounting""Claus_output"
Chainfilter"Accounting""Pavel_output"
Chainfilter"Accounting""Perry_output"
Chainfilter"Accounting""etcd_client_output"
Chainfilter"Accounting""etcd_cluster_output"
</Plugin>
Collectd-Networkusage
138
Nowwerestartcollectdwith/etc/init.d/collectdrestart,watchthesyslogforerrors.IfeverythingisOK,ourvaluesshouldshowupin:
/var/lib/collectd/rrd/localhost/iptables-filter-Accounting/ipt_packets-Claus_output.rrd
Wecaninspectourvalueswithkcollectd:
Author:WilfriedGoesgens
Tags:#monitoring
Collectd-Networkusage
139
MonitoringotherrelevantmetricsofArangoDB
Problem
AsideofthevalueswhichArangoDBalreadyoffersformonitoring,othersystemmetricsmayberelevantforcontinuouslyoperatingArangoDB.beitasingleinstanceoraclustersetup.Collectdoffersapleathoraofplugins-letshavealookatsomeofthemwhichmaybeusefulforus.
Solution
Ingedients
Forthisrecipeyouneedtoinstallthefollowingtools:
collectd:ThemetricsaggregationDaemonwebaseonMonitoringwithCollecdrecipeforunderstandingthebasicsaboutcollectd
Diskusage
YoumaywanttomonitorthatArangoDBdoesn'trunoutofdiskspace.ThedfPlugincanaggregatethesevaluesforyou.
FirstweneedtofindoutwhichdisksareusedbyyourArangoDB.Bydefaultyouneedtofind/var/lib/arangointhemountpoints.Sincenowadaysmanyvirtualfilesystemsarealsomountedonatypical*nixsystemwewanttosorttheoutputofmount:
mount|sort
/dev/sda3on/local/hometypeext4(rw,relatime,data=ordered)
/dev/sda4on/typeext4(rw,relatime,data=ordered)
/dev/sdb1on/mnttypevfat(rw,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=utf8,shortname=mixed,errors=remount-ro)
binfmt_miscon/proc/sys/fs/binfmt_misctypebinfmt_misc(rw,relatime)
cgroupon/sys/fs/cgroup/blkiotypecgroup(rw,nosuid,nodev,noexec,relatime,blkio)
....
udevon/devtypedevtmpfs(rw,relatime,size=10240k,nr_inodes=1022123,mode=755)
Soherewecanseethemountpointsare/,/local/home,/mnt/so/var/lib/canbefoundontherootpartition(/)/dev/sda3here.AproductionsetupmaybedifferentsotheOSdoesn'tinterferewiththeservices.
Thecollectdconfiguration/etc/collectd/collectd.conf.d/diskusage.conflookslikethis:
LoadPlugindf
<Plugindf>
Device"/dev/sda3"
#Device"192.168.0.2:/mnt/nfs"
#MountPoint"/home"
#FSType"ext4"
#ignorerootfs;else,therootfile-systemwouldappeartwice,causing
#oneoftheupdatestofailandspamthelog
FSTyperootfs
#ignoretheusualvirtual/temporaryfile-systems
FSTypesysfs
FSTypeproc
FSTypedevtmpfs
FSTypedevpts
FSTypetmpfs
FSTypefusectl
FSTypecgroup
IgnoreSelectedtrue
#ReportByDevicefalse
#ReportReservedfalse
#ReportInodesfalse
#ValuesAbsolutetrue
#ValuesPercentagefalse
</Plugin>
Collectd-moreMetrics
140
DiskI/OUsage
Anotherinterestingmetricistheamountofdataread/writtentodisk-itsanestimatehowbusyyourArangoDBorthewholesystemcurrentlyis.TheDiskpluginaggregatesthesevalues.
Accordingtothemountpointsaboveourconfiguration/etc/collectd/collectd.conf.d/disk_io.conflookslikethis:
LoadPlugindisk
<Plugindisk>
Disk"hda"
Disk"/sda[23]/"
IgnoreSelectedfalse
</Plugin>
CPUUsage
WhiletheArangoDBselfmonitoringalreadyofferssomeoverviewoftherunningthreadsetc.youcangetadeeperviewusingtheProcessPlugin.
Ifyou'rerunningasingleArangoinstance,asimplematchbyprocessnameissufficient,/etc/collectd/collectd.conf.d/arango_process.conflookslikethis:
LoadPluginprocesses
<Pluginprocesses>
Process"arangod"
</Plugin>
Ifyou'rerunningacluster,youcanmatchthespecificinstancesbycommand-lineparameters,/etc/collectd/collectd.conf.d/arango_cluster.conflookslikethis:
LoadPluginprocesses
<Pluginprocesses>
ProcessMatch"Claus""/usr/bin/arangod.*--cluster.my-idClaus.*"
ProcessMatch"Pavel""/usr/bin/arangod.*--cluster.my-idPavel.*"
ProcessMatch"Perry""/usr/bin/arangod.*--cluster.my-idPerry.*"
Process"etcd-arango"
</Plugin>
MorePlugins
Asmentionedabove,thelistofavailablepluginsishuge;Herearesomemoreonecouldbeinterestedin:
usetheCPUPlugintomonitortheoverallCPUutilizationusetheMemoryPlugintomonitormainmemoryavailabilityusetheSwapPlugintoseewhetherexcessRAMusageforcesthesystemtopageandthusslowdownEthernetStatisticswithwhatsgoingonatyourNetworkcardstogetamorebroadoverviewofnetworktrafficyoumayTaillogfileslikeanapacherequestlogandpickspecificrequestsbyregularexpressionsParsetabularfilesinthe/procfilesystemyoucanusefilterstoreducetheamountofdatacreatedbyplugins(i.e.ifyouhavemanyCPUcores,youmaywantthecombinedresult).Itcanalsodecidewheretoroutedataandtowhichwriterpluginwhileyoumayhaveseenthatmetricsarestoredatafixedrateorfrequency,yourmetrics(i.e.thedurationsofwebrequests)maycomeinarandom&higherfrequency.Thusyouwanttoburnthemdowntoafixedfrequency,andknowMin/Max/Average/Median.SoyouwanttoAggregatevaluesusingthestatsdpattern.YoumaystartrollingyourowninPython,java,PerlorforsureinC,thelanguagecollectdisimplementedin
Finallywhilekcollectdisnicetogetaquicksuccessatinspectingyourcollectedmetricsduringworkingyourwayintocollectd,itsnotassufficientforoperatingaproductionsite.SincecollectdsdefaultstorageRRDisalreadywidespreadinsystemmonitoring,therearemanywebfrontentstochooseforthevisualization.SomeofthemreplacetheRRDstoragebysimplyaddingawriterplugin,mostprominenttheGraphitegraphingframeworkwiththeGraphitewriterwhichallowsyoutocombinerandommetricsinsinglegraphs-tofindcoincidencesinyourdatayouneverdreamedof.
IfyoualreadyrunNagiosyoucanusetheNagiostooltosubmitvalues.
Collectd-moreMetrics
141
Wehopeyounowhaveagoodoverviewofwhatspossible,butasusualitsagoodideatobrowsetheFineManual.
Author:WilfriedGoesgens
Tags:#monitoring
Collectd-moreMetrics
142
MonitoringyourFoxxapplicationsNote:thisrecipeisworkingwithArangoDB2.5Foxx
Problem
HowtointegrateaFoxxapplicationintoamonitoringsystemusingthecollectdcurl_JSONplugin.
Solution
SinceFoxxnativetongueisJSON,integratingitwiththecollectdcurl_JSONpluginshouldbeaneasyexercise.WehaveaFoxx-ApplicationwhichcanreceiveDataandwriteitintoacollection.WespecifyaneasyinputModel:
Model=Foxx.Model.extend({
schema:{
//DescribetheattributeswithJoihere
'_key':Joi.string(),
'value':Joi.number()
}
});
AnduseasimpleFoxx-Routetoinjectdataintoourcollection:
/**CreatesanewFirstCollection
*
*CreatesanewFirstCollection-Item.Theinformationhastobeinthe
*requestBody.
*/
controller.post('/firstCollection',function(req,res){
varfirstCollection=req.params('firstCollection');
firstCollection.attributes.Date=Date.now();
res.json(FirstCollection_repo.save(firstCollection).forClient());
}).bodyParam('firstCollection',{
description:'TheFirstCollectionyouwanttocreate',
type:FirstCollection
});
WhichwemaydousingcURL:
echo'{"value":1,"_key":"13"}'|\
curl-d@-http://localhost:8529/_db/_system/collectable_foxx/data/firstCollection/firstCollection
We'dexpectthevaluetobeintherangeof1to5.Maybethesourceofthisdataisaweb-pollorsomethingsimilar.
WenowaddanotherFoxx-routewhichwewanttolinkwithcollectd:
/**
*weuseagroup-byconstructtogetthevalues:
*/
vardb=require('org/arangodb').db;
varsearchQuery='FORxIN@@collectionFILTERx.Date>=@untilcollectvalue=x.valuewithcountintocounterRETURN{[[CONCAT
("choice",value)]:counter}';
controller.get('/firstCollection/lastSeconds/:nSeconds',function(req,res){
varuntil=Date.now()-req.params('nSeconds')*1000;
res.json(
db._query(searchQuery,{
'@collection':FirstCollection_repo.collection.name(),
'until':until
}).toArray()
);
}).pathParam('nSeconds',{
description:'lookuptonSecondsintothepast',
Collectd-MonitoringFoxx
143
type:joi.string().required()
});
Weinspectthereturndocumentusingcurlandjqforniceformatting:
curl'http://localhost:8529/_db/_system/collectable_foxx/data/firstCollection/firstCollection/lastSeconds/10'|jq"."
[
{
"1":3
"3":7
}
]
Wehavetodesignthereturnvaluesinawaythatcollectd'sconfigsyntaxcansimplygrabit.ThisRoutereturnsanobjectwithflatkeyvalueswherekeysmayrangefrom0to5.Wecreateasimplecollectdconfigurationin/etc/collectd/collectd.conf.d/foxx_simple.confthatmatchesourAPI:
#Loadtheplug-in:
LoadPlugincurl_json
#weneedtouseourowntypestogenerateindividualnamesforourgauges:
TypesDB"/etc/collectd/collectd.conf.d/foxx_simple_types.db"
<Plugincurl_json>
#AdjusttheURLsocollectdcanreachyourarangod:
<URL"http://localhost:8529/_db/_system/collectable_foxx/data/firstCollection/firstCollection/lastSeconds/10">
#SetyourauthenticationtoAardvarkhere:
#User"foo"
#Password"bar"
<Key"choice0">
Type"the_values"
</Key>
<Key"choice1">
Type"first_values"
</Key>
<Key"choice2">
Type"second_values"
</Key>
<Key"choice3">
Type"third_values"
</Key>
<Key"choice4">
Type"fourth_values"
</Key>
<Key"choice5">
Type"fifth_values"
</Key>
</URL>
</Plugin>
Togetnicemetricnames,wespecifyourowntypes.dbfilein/etc/collectd/collectd.conf.d/foxx_simple_types.db:
the_valuesvalue:GAUGE:U:U
first_valuesvalue:GAUGE:U:U
second_valuesvalue:GAUGE:U:U
third_valuesvalue:GAUGE:U:U
fourth_valuesvalue:GAUGE:U:U
fifth_valuesvalue:GAUGE:U:U
Author:WilfriedGoesgens
Tags:#monitoring#foxx#json
Collectd-MonitoringFoxx
144