Browse Source

simplifying database

master
Erwan Glasziou 8 months ago
parent
commit
ab60207f74
  1. 33
      database-update.py

33
database-update.py

@ -21,7 +21,7 @@ response = urlopen(url)
# from url in data
data_json = json.loads(response.read())
bulk_files = [];
bulk_files = []
# print the json response
for file in data_json["data"]:
@ -51,15 +51,42 @@ for file in bulk_files:
#json downloaded, moving to db
print("Updating cards.db ...\n")
numcard = 0
numlayout = 0
with open(filename, "r") as f:
rawjson = naya.stream_array(naya.tokenize(f))
batch = []
for card in rawjson:
if card["lang"] in ['fr', 'en']:
if card["layout"] in ["token","art_series", "double_faced_token", "planar", "scheme", "vanguard", "emblem"]:
numlayout+=1
elif ((card["lang"] in ['fr', 'en']) and card["digital"] == 0):
#print(card.keys())
card.pop("oversized")
card.pop("digital")
card.pop("object")
if "tcgplayer_id" in card.keys():
card.pop("tcgplayer_id")
if "tcgplayer_etched_id" in card.keys():
card.pop("tcgplayer_etched_id")
if "arena_id" in card.keys():
card.pop("arena_id")
if "mtgo_id" in card.keys():
card.pop("mtgo_id")
if "mtgo_foil_id" in card.keys():
card.pop("mtgo_foil_id")
if "cardmarket_id" in card.keys():
card.pop("cardmarket_id")
if "multiverse_id" in card.keys():
card.pop("multiverse_id")
if "security_stamp" in card.keys():
card.pop("security_stamp")
card.pop("booster")
card.pop("games")
card.pop("set_type")
batch.append(card)
if len(batch) == 512:
if len(batch) == 512:
cards.insert_all(batch,alter=True,pk="id",batch_size=512)
batch = []
print("Dropped {} cards because of layout \r".format(numlayout))
sys.stdout.write("Adding card {} \r".format(numcard))
numcard+=1
print("\n")
Loading…
Cancel
Save