simplifying database

database-update.py

@@ -21,7 +21,7 @@ response = urlopen(url)
 # from url in data
 data_json = json.loads(response.read())
 
-bulk_files = [];
+bulk_files = []
 
 # print the json response
 for file in data_json["data"]:
@@ -51,15 +51,42 @@ for file in bulk_files:
     #json downloaded, moving to db
     print("Updating cards.db ...\n")
     numcard = 0
+    numlayout = 0
     with open(filename, "r") as f:
         rawjson = naya.stream_array(naya.tokenize(f))
         batch = []
         for card in rawjson:
-            if card["lang"] in ['fr', 'en']:
+            if card["layout"] in ["token","art_series", "double_faced_token", "planar", "scheme", "vanguard", "emblem"]:
+                numlayout+=1
+            elif ((card["lang"] in ['fr', 'en']) and card["digital"] == 0):
+                #print(card.keys())
+                card.pop("oversized")
+                card.pop("digital") 
+                card.pop("object")
+                if "tcgplayer_id" in card.keys():
+                    card.pop("tcgplayer_id")
+                if "tcgplayer_etched_id" in card.keys():
+                    card.pop("tcgplayer_etched_id")
+                if "arena_id" in card.keys():
+                    card.pop("arena_id")
+                if "mtgo_id" in card.keys():
+                    card.pop("mtgo_id")
+                if "mtgo_foil_id" in card.keys():
+                    card.pop("mtgo_foil_id")
+                if "cardmarket_id" in card.keys():
+                    card.pop("cardmarket_id")
+                if "multiverse_id" in card.keys():
+                    card.pop("multiverse_id")
+                if "security_stamp" in card.keys():
+                    card.pop("security_stamp")
+                card.pop("booster")
+                card.pop("games")
+                card.pop("set_type")
                 batch.append(card)
-            if len(batch) == 512:
+            if len(batch) == 512: 
                 cards.insert_all(batch,alter=True,pk="id",batch_size=512)
                 batch = []
+                print("Dropped {} cards because of layout \r".format(numlayout))
                 sys.stdout.write("Adding card {} \r".format(numcard))
             numcard+=1
     print("\n")