second BerkeleyDB example
"""
-A simple example showing how to use a BerkeleyDB store to do on-disk persistence:
+BerkeleyDB in use as a persistent Graph store.
+
+Example 1: simple actions
* creating a ConjunctiveGraph using the BerkeleyDB Store
* adding triples to it
@@ -8,15 +10,20 @@ A simple example showing how to use a BerkeleyDB store to do on-disk persistence
* re-opening the store using the same DB files
* getting the same count of triples as before
-"""
+Example 2: larger data
+* loads multiple graphs downloaded from GitHub into a BerkeleyDB-baked graph stored in the folder gsq_vocabs.
+* does not delete the DB at the end so you can see it on disk
+"""
+import os
from rdflib import ConjunctiveGraph, Namespace, Literal
from rdflib.store import NO_STORE, VALID_STORE
-
from tempfile import mktemp
-if __name__ == "__main__":
+def example_1():
+ """Creates a ConjunctiveGraph and performs some BerkeleyDB tasks with it
+ """
path = mktemp()
# Declare we are using a BerkeleyDB Store
@@ -68,8 +75,60 @@ if __name__ == "__main__":
graph.close()
# Clean up the temp folder to remove the BerkeleyDB database files...
- import os
-
for f in os.listdir(path):
os.unlink(path + "/" + f)
os.rmdir(path)
+
+
+def example_2():
+ """Loads a number of SKOS vocabularies from GitHub into a BerkeleyDB-backed graph stored in the local folder
+ 'gsq_vocabs'
+
+ Should print out the number of triples after each load, e.g.:
+ 177
+ 248
+ 289
+ 379
+ 421
+ 628
+ 764
+ 813
+ 965
+ 1381
+ 9666
+ 9719
+ ...
+ """
+ from urllib.request import urlopen, Request
+ from urllib.error import HTTPError
+ import json
+ import base64
+
+ g = ConjunctiveGraph("BerkeleyDB")
+ g.open("gsg_vocabs", create=True)
+
+ # gsq_vocabs = "https://api.github.com/repos/geological-survey-of-queensland/vocabularies/git/trees/master"
+ gsq_vocabs = "https://api.github.com/repos/geological-survey-of-queensland/vocabularies/git/trees/cd7244d39337c1f4ef164b1cf1ea1f540a7277db"
+ try:
+ res = urlopen(Request(gsq_vocabs, headers={"Accept": "application/json"}))
+ except HTTPError as e:
+ return e.code, str(e), None
+
+ data = res.read()
+ encoding = res.info().get_content_charset('utf-8')
+ j = json.loads(data.decode(encoding))
+ for v in j["tree"]:
+ # process the element in GitHub result if it's a Turtle file
+ if v["path"].endswith(".ttl"):
+ # for each file, call it by URL, decode it and parse it into the graph
+ r = urlopen(v['url'])
+ content = json.loads(r.read().decode())["content"]
+ g.parse(data=base64.b64decode(content).decode(), format="turtle")
+ print(len(g))
+
+ print("loading complete")
+
+
+if __name__ == "__main__":
+ example_1()
+ example_2()
