Browse Source

it works

master
Dorian Bourgeoisat 9 months ago
parent
commit
84102a2df8
  1. 2
      .gitignore
  2. 29
      database-update.py

2
.gitignore

@ -1,3 +1,3 @@
/target
cards.db
cards.db*
all_cards*

29
database-update.py

@ -1,10 +1,15 @@
#!/bin/python
import sys
# import urllib library
from urllib.request import urlopen
# import json
import json
import naya
# import sqlite utils library
from sqlite_utils import Database
# store the URL in url as
# parameter for urlopen
url = "https://api.scryfall.com/bulk-data"
@ -28,10 +33,14 @@ for file in data_json["data"]:
CHUNK = 256 * 1024
db = Database("cards.db", recreate=True)
db.enable_wal()
cards = db["cards"]
for file in bulk_files:
uri = file["download_uri"]
filename = file["type"]+"_"+file["id"]+".json"
print("Downloading {} ...".format(uri))
rep = urlopen(uri)
with open(filename, 'wb') as f:
while True:
@ -39,4 +48,16 @@ for file in bulk_files:
if not chunk:
break
f.write(chunk)
print("Updating cards.db ...\n")
numcard = 0
with open(filename, "r") as f:
rawjson = naya.stream_array(naya.tokenize(f))
batch = []
for card in rawjson:
batch.append(card)
if len(batch) == 512:
cards.insert_all(batch,alter=True,pk="id",batch_size=512)
batch = []
sys.stdout.write("Adding card {} \r".format(numcard))
numcard+=1
print("\n")
Loading…
Cancel
Save