-
Notifications
You must be signed in to change notification settings - Fork 4
/
main.py
397 lines (333 loc) · 22.2 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
import os
import pathlib
import requests
from bs4 import BeautifulSoup, Tag, NavigableString
import card_structure
import helpers
import copy
import re
from alive_progress import alive_bar
from langdetect import detect
import time
import argparse
import sys
YGOPRODECK_API = "https://db.ygoprodeck.com/api/v7/cardinfo.php"
link_dict = {1: "Bottom-Left",
2: "Bottom",
3: "Bottom-Right",
4: "Left",
6: "Right",
7: "Top-Left",
8: "Top",
9: "Top-Right"}
if __name__ == "__main__":
# ARGUMENTS PARSING #
parser = argparse.ArgumentParser()
parser.add_argument("-l", "--language", action="store", type=str,
choices=["en", "fr", "ja", "de", "it", "es", "pt", "ko"],
default="en",
help="Select the generation language")
parser.add_argument("-f", "--fast", action='store_true', default=False)
args = parser.parse_args()
###########################
script_dir = pathlib.Path(__file__).parent.resolve()
# Check Platform
helpers.check_platform()
# Clone the yugioh-card-history git
repository_url = 'https://github.com/db-ygorganization-com/yugioh-card-history.git'
clone_repo_path = pathlib.PurePath(script_dir, "yugioh-card-history")
helpers.clone_or_pull_repo(repository_url, str(clone_repo_path))
# English="en", French="fr", Japanese="ja", Deutsch="de", Italian="it", Spanish="es", Portugese = "pt", Korean="ko"
language_code = args.language
scrape_additional_data = not args.fast
# Get the language database
file_pattern = pathlib.PurePath(script_dir, "yugioh-card-history", language_code, "*.json")
card_hashmap = helpers.combine_json_files(str(file_pattern), language_code, 'name')
card_en_id_hashmap = helpers.combine_json_files(str(file_pattern), 'en', 'id')
delimiter = '$'
output_path = pathlib.PurePath(script_dir, "data", language_code)
wiki_URL = "https://yugioh.fandom.com/wiki/"
base_URL = "https://www.db.yugioh-card.com"
URL = f"https://www.db.yugioh-card.com/yugiohdb/card_list.action??clm=1&wname=CardSearch&request_locale={language_code}"
pathlib.Path(output_path).mkdir(parents=True, exist_ok=True)
# Initiate a session and update the headers.
requests_session = requests.session()
headers = {
'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36',
}
requests_session.headers.update(headers)
# Initialization
list_of_cards = []
processed_card_database = {}
count = 0
pack_elements, link_elements, pack_names = helpers.get_packs_and_links(URL, requests_session)
with alive_bar(len(link_elements), dual_line=True, title='Packs Processed', force_tty=True) as bar:
for element_num in range(len(link_elements)): # Loop through Packs
# If the count exceeds number of names - Exit
if count >= len(pack_names):
break
# Get name of the Pack from the pack HTML array
pack_name = helpers.get_pack_name(pack_names, count)
count += 1 # Increase the count for next iteration
if os.path.isfile(f"{output_path}/{pack_name}.csv"):
bar()
continue
# Get the link for each individual pack
card_elements, raw_url1 = helpers.get_card_elements(base_URL, link_elements[element_num], language_code, requests_session)
bar.text = f'-> Processing pack: {pack_name}, please wait...'
# Looping through all the cards on the page
card_count = 0
for card_info in card_elements:
card_count += 1
tmp_card = card_structure.Card()
name = card_info.find_all("span", class_="card_name")
tmp_card.name = name[0].text
if name[0].text in processed_card_database:
list_of_cards.append(copy.copy(processed_card_database[name[0].text]))
continue
# #Uncomment this code if you need to debug a certain card
# if name[0].text == "Enemy Controller":
# print("e")
card_start_time = time.time()
card_hash_data = card_hashmap.get(name[0].text)
process_english_name = False
if card_hash_data != None:
if card_hash_data.get('localizedAttribute') != None:
tmp_card.attribute = card_hash_data.get('localizedAttribute')
elif card_hash_data.get('attribute') != None:
tmp_card.attribute = card_hash_data.get('attribute')
elif card_hash_data.get('frameType') != None:
tmp_card.attribute = card_hash_data.get('frameType').upper()
if card_hash_data.get('englishAttribute') == "trap" \
or card_hash_data.get('frameType') == "trap" \
or card_hash_data.get('englishAttribute') == "spell" \
or card_hash_data.get('frameType') == "spell":
if card_hash_data.get('localizedProperty') != None:
tmp_card.spell_attribute = card_hash_data.get('localizedProperty')
elif card_hash_data.get('race') != None:
tmp_card.spell_attribute = card_hash_data.get('race')
if card_hash_data.get('desc') != None:
tmp_card.card_text = card_hash_data.get('desc').replace('\r\n', ' ').replace('\n', ' ')
else:
tmp_card.card_text = card_hash_data.get('effectText').replace('\r\n', ' ').replace('\n', ' ')
else: # It's a monster
if card_hash_data.get('level') != None:
tmp_card.level = card_hash_data.get('level')
if card_hash_data.get('rank') != None:
tmp_card.rank = card_hash_data.get('rank')
if card_hash_data.get('linkRating') != None:
tmp_card.link = card_hash_data.get('linkRating')
elif card_hash_data.get('linkval') != None:
tmp_card.link = card_hash_data.get('linkval')
if card_hash_data.get('linkArrows') != None:
tmp_card.link_arrows = '/'.join(
[link_dict[int(key)] for key in card_hash_data.get('linkArrows')])
tmp_card.link_arrows = tmp_card.link_arrows.split('/')
elif card_hash_data.get('linkmarkers') != None:
tmp_card.link_arrows = card_hash_data.get('linkmarkers')
if card_hash_data.get('pendScale') != None:
tmp_card.pend_scale = card_hash_data.get('pendScale')
elif card_hash_data.get('scale') != None:
tmp_card.pend_scale = card_hash_data.get('scale')
if tmp_card.pend_scale != "":
if card_hash_data.get('pendEffect') != None:
tmp_card.pend_effect = card_hash_data.get('pendEffect')
tmp_card.card_text = card_hash_data.get('effectText')
elif card_hash_data.get('desc') != None:
pattern = r"\[ Pendulum Effect \](.*?)\[ Monster Effect \](.*?)$"
matches = re.search(pattern, card_hash_data.get('desc'), re.DOTALL)
if matches:
tmp_card.pend_effect = matches.group(1).replace('\n', '').replace('\r', '')
if '\n' in matches.group(2):
tmp_card.summoning_condition = matches.group(2).split('\n')[0].replace('\r', '')
tmp_card.card_text = matches.group(2).split('\n')[1]
else:
tmp_card.card_text = matches.group(2).replace('\r', '').replace('\n', '')
elif card_hash_data.get('desc') != None:
if '\n' in card_hash_data.get('desc'):
tmp_card.summoning_condition = card_hash_data.get('desc').split('\n')[0].replace('\r', '')
tmp_card.card_text = card_hash_data.get('desc').split('\n')[1].replace('\n', '').replace('\r', '')
else:
tmp_card.card_text = card_hash_data.get('desc').replace('\n', '').replace('\r', '')
if card_hash_data.get('properties') != None:
tmp_card.type = '/'.join(card_hash_data.get('properties'))
else:
tmp_card.type = f"[{card_hash_data.get('race')}/{card_hash_data.get('frameType')}]"
if card_hash_data.get('atk') != None:
tmp_card.attack = card_hash_data.get('atk')
if card_hash_data.get('def') != None:
tmp_card.defense = card_hash_data.get('def')
if card_hash_data.get('effectText') != None and '\n' in card_hash_data.get('effectText'):
tmp_card.summoning_condition = card_hash_data.get('effectText').split('\n')[0]
# Use the konami code to find other information in ygoprodeck api
try:
en_hashmap_data = card_en_id_hashmap.get(card_hash_data.get('misc_info')[0].get('konami_id'))
except:
en_hashmap_data = None
if en_hashmap_data != None:
if en_hashmap_data.get('name') != None:
tmp_card.card_passcode = en_hashmap_data.get('id')
english_name = en_hashmap_data.get('name')
else:
process_english_name = True
else:
process_english_name = True
if process_english_name == True:
if language_code != "en":
link_value_struct = card_info.find_all("input", class_="link_value")
link_value = link_value_struct[0].attrs['value']
card_info_page, card_raw_url = helpers.get_page(base_URL + link_value, language_code, requests_session)
try:
english_name = card_info_page.find("div", {"id": "cardname"}).h1
english_name = english_name.select_one("span").text
except:
continue
if len(english_name) > 2:
if detect(english_name) != 'en':
continue
else:
english_name = tmp_card.name
attribute = card_info.find_all("span", class_="box_card_attribute")
attribute = helpers.cleanStr(attribute[0].text, [("\n", "")])
tmp_card.attribute = attribute
# Monster card specific information
# Exemption list is missing japanese and korean so will not work for those two languages and will probably crash
exemption_list = ['SPELL', 'TRAP', 'MAGIE', 'PIÈGE', 'ZAUBER', 'FALLE', 'MAGIA', 'TRAPPOLA', 'MÁGICA', 'TRAMPA', 'ARMADILHA', 'MAGIA', '魔法', '罠', '마법', '함정']
monster_card_found = True
for pattern in exemption_list:
if re.search(pattern, attribute):
monster_card_found = False
if monster_card_found:
level = card_info.find_all("span", class_="box_card_level_rank level")
if len(level) != 0:
level = helpers.cleanStr(level[0].text, [("Level ", ""), ("\n", "")])
tmp_card.level = level
rank = card_info.find_all("span", class_="box_card_level_rank rank")
if len(rank) != 0:
rank = helpers.cleanStr(rank[0].text, [("Rank ", ""), ("\n", "")])
tmp_card.rank = rank
link = card_info.find_all("span", class_="box_card_linkmarker")
if len(link) != 0:
link = helpers.cleanStr(link[0].text, [("Link ", ""), ("\n", "")])
tmp_card.link = link
card_type = card_info.find_all("span", class_="card_info_species_and_other_item")
card_type = helpers.cleanStr(card_type[0].text, [("\n", ""), ("\t", ""), ("\r", "")])
tmp_card.type = card_type
if re.search('Pendulum', card_type):
pend_scale = card_info.find_all("span", class_="box_card_pen_scale")
pend_scale = helpers.cleanStr(pend_scale[0].text, [("P Scale ", ""), ("\n", ""), ("\t", ""), ("\r", "")])
tmp_card.pend_scale = pend_scale
pend_effect = card_info.find_all("span", class_="box_card_pen_effect c_text flex_1")
pend_effect = helpers.cleanStr(pend_effect[0].text, [("\n", ""), ("\t", ""), ("\r", "")])
tmp_card.pend_effect = pend_effect
attack_power = card_info.find_all("span", class_="atk_power")
attack_power = helpers.cleanStr(attack_power[0].text, [("ATK ", ""), ("\n", ""), ("\t", ""), ("\r", "")])
tmp_card.attack = attack_power
def_power = card_info.find_all("span", class_="def_power")
def_power = helpers.cleanStr(def_power[0].text, [("DEF ", ""), ("\n", ""), ("\t", ""), ("\r", "")])
tmp_card.defense = def_power
# Spell and Trap card specific information
if re.search('SPELL', attribute) or re.search('TRAP', attribute):
spell_attribute = card_info.find_all("span", class_="box_card_effect")
if len(spell_attribute) != 0:
spell_attribute = helpers.cleanStr(spell_attribute[0].text, [("\n", ""), ("\t", ""), ("\r", "")])
tmp_card.spell_attribute = spell_attribute
card_text = card_info.find_all("dd", class_="box_card_text c_text flex_1")
if len(card_text[0].contents) > 1 and ("Fusion" in tmp_card.type or "Synchro" in tmp_card.type
or "Xyz" in tmp_card.type or "Link" in tmp_card.type):
summoning_condition = helpers.cleanStr(card_text[0].contents[0], [("\n", ""), ("\t", ""), ("\r", "")])
new_card_text = ""
for i in range(1, len(card_text[0].contents)):
if isinstance(card_text[0].contents[i], Tag):
continue
new_card_text += helpers.cleanStr(card_text[0].contents[i], [("\n", ""), ("\t", ""), ("\r", "")])
card_text = new_card_text
else:
summoning_condition = ""
card_text = helpers.cleanStr(card_text[0].text, [("\n", ""), ("\t", ""), ("\r", "")])
tmp_card.summoning_condition = summoning_condition
tmp_card.card_text = card_text
# Grab additional Card information from the wiki
extra_info_time_start = time.time()
if scrape_additional_data:
processed_card_name = helpers.process_english_name(english_name)
debug_url = wiki_URL + processed_card_name
card_url = requests_session.get(wiki_URL + processed_card_name) # Request the URL containing the card lists
if card_url.status_code != 200:
print(f"error occured for card {tmp_card.name} : {english_name}: url: {debug_url}")
soup_deck = BeautifulSoup(card_url.content, "html.parser") # Pass through html parser
ind_card_elements = soup_deck.find_all("tr", class_="cardtablerow") # Find all card structures
if card_url.status_code == 200 and len(ind_card_elements) == 0:
# We'll try specifying (card) in the name
card_url = requests_session.get(wiki_URL + processed_card_name + "_(card)") # Request the URL containing the card lists
soup_deck = BeautifulSoup(card_url.content, "html.parser") # Pass through html parser
ind_card_elements = soup_deck.find_all("tr", class_="cardtablerow") # Find all card structures
if len(ind_card_elements) == 0:
print(f"error occured for card {tmp_card.name} : {english_name} - failed with (card): url: {debug_url}")
for ind_card_info in ind_card_elements:
if re.search('Passcode', ind_card_info.text):
lists = ind_card_info.find_all("td", class_="cardtablerowdata")
if len(lists) != 0:
passcode = lists[0]
tmp_card.card_passcode = helpers.cleanStr(passcode.text, [("\n", ""), ("\t", ""), ("\r", "")])
if re.search('Link Arrows', ind_card_info.text):
lists = ind_card_info.find_all("td", class_="cardtablerowdata")
if len(lists) != 0:
card_arrows = lists[0]
arrows = []
for arrow in card_arrows:
if not isinstance(arrow, NavigableString) and arrow.text != '':
arrows.append(arrow.text)
tmp_card.link_arrows = arrows
if re.search('Card effect types', ind_card_info.text):
lists = ind_card_info.find_all("td", class_="cardtablerowdata")
if len(lists) != 0:
card_effect_types = lists[0]
effect_types = []
for effect_type in card_effect_types:
if not isinstance(effect_type, NavigableString) and effect_type.text != '\n':
for effect in effect_type:
if effect.text not in effect_types and not re.search('\n', effect.text):
effect_types.append(effect.text)
tmp_card.effect_types = effect_types
if re.search('Statuses', ind_card_info.text):
lists = ind_card_info.find_all("td", class_="cardtablerowdata")
if len(lists) != 0:
card_status = lists[0]
tmp_card.card_status = helpers.cleanStr(card_status.text, [(" ", "")])
if re.search('Card search categories', ind_card_info.text):
lists = ind_card_info.find_all("div", class_="hlist")
for row_list in lists:
if re.search('Supports', row_list.text):
card_supports = []
card_search_categories = row_list.contents[1]
for support in card_search_categories:
if not '\n' in support.text and support.text != 'Supports ':
card_supports.append(support.text)
tmp_card.card_supports = card_supports
if re.search('Anti-supports', row_list.text):
card_anti_supports = []
card_search_categories = row_list.contents[1]
for anti_support in card_search_categories:
if not '\n' in anti_support.text and anti_support.text != 'Anti-supports ':
card_anti_supports.append(anti_support.text)
tmp_card.card_anti_supports = card_anti_supports
if re.search('Actions', row_list.text):
card_actions = []
card_search_categories = row_list.contents[1]
for action in card_search_categories:
if not re.search('\n', action.text) and action.text != 'Actions ':
card_actions.append(action.text)
tmp_card.card_actions = card_actions
card_end_time = time.time()
#print(f"Card Process Time: {card_end_time - card_start_time} - Extra Info Time: {card_end_time - extra_info_time_start}")
bar.text = f'-> Processing pack: {pack_name}, {tmp_card.name}'
# Store the card structure into the list of cards
list_of_cards.append(copy.copy(tmp_card))
processed_card_database[tmp_card.name] = copy.copy(tmp_card)
tmp_card.clear() # Clear the individual temporary structure for next card
helpers.outputCSV(f"{output_path}/{pack_name}.csv", list_of_cards, delimiter)
list_of_cards.clear() # Reset list for the next pack
bar()
print("DONE")