You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

92 lines
2.9 KiB

#!/bin/python
import sys
# import urllib library
from urllib.request import urlopen
# import json
import json
import naya
# import sqlite utils library
from sqlite_utils import Database
# store the URL in url as
# parameter for urlopen
url = "https://api.scryfall.com/bulk-data"
# store the response of URL
response = urlopen(url)
# storing the JSON response
# from url in data
data_json = json.loads(response.read())
bulk_files = []
# print the json response
for file in data_json["data"]:
if (file["type"]=="all_cards"):
print("{} <- Will download this database".format(file["name"]))
bulk_files.append(file)
else:
print("{}".format(file["name"]))
CHUNK = 256 * 1024
db = Database("cards.db", recreate=True)
db.enable_wal()
cards = db["cards"]
for file in bulk_files:
uri = file["download_uri"]
filename = file["type"]+"_"+file["id"]+".json"
print("Downloading {} ...".format(uri))
rep = urlopen(uri)
with open(filename, 'wb') as f:
while True:
chunk = rep.read(CHUNK)
if not chunk:
break
f.write(chunk)
#json downloaded, moving to db
print("Updating cards.db ...\n")
numcard = 0
numlayout = 0
with open(filename, "r") as f:
rawjson = naya.stream_array(naya.tokenize(f))
batch = []
for card in rawjson:
if card["layout"] in ["token","art_series", "double_faced_token", "planar", "scheme", "vanguard", "emblem"]:
numlayout+=1
elif ((card["lang"] in ['fr', 'en']) and card["digital"] == 0):
#print(card.keys())
card.pop("oversized")
card.pop("digital")
card.pop("object")
if "tcgplayer_id" in card.keys():
card.pop("tcgplayer_id")
if "tcgplayer_etched_id" in card.keys():
card.pop("tcgplayer_etched_id")
if "arena_id" in card.keys():
card.pop("arena_id")
if "mtgo_id" in card.keys():
card.pop("mtgo_id")
if "mtgo_foil_id" in card.keys():
card.pop("mtgo_foil_id")
if "cardmarket_id" in card.keys():
card.pop("cardmarket_id")
if "multiverse_id" in card.keys():
card.pop("multiverse_id")
if "security_stamp" in card.keys():
card.pop("security_stamp")
card.pop("booster")
card.pop("games")
card.pop("set_type")
batch.append(card)
if len(batch) == 512:
cards.insert_all(batch,alter=True,pk="id",batch_size=512)
batch = []
print("Dropped {} cards because of layout \r".format(numlayout))
sys.stdout.write("Adding card {} \r".format(numcard))
numcard+=1
print("\n")