Refactor dataset build

This commit is contained in:
hoguchi live 2022-10-22 22:18:52 -10:00
parent 086370a654
commit 8aed9f49f5
9 changed files with 260 additions and 487 deletions

@ -1 +0,0 @@
Subproject commit 8a44c3480a9659449fae46d55fb966ce1a410a33

View File

@ -13,7 +13,7 @@
<link rel="stylesheet" type="text/css" href="src/css/styles.css"> <link rel="stylesheet" type="text/css" href="src/css/styles.css">
<script src="src/js/data.js"></script> <script src="src/js/data.js"></script>
<script src="src/js/data/2022-05-10.js"></script> <script src="src/js/data/data.js"></script>
<script src="src/js/html2canvas.min.js"></script> <script src="src/js/html2canvas.min.js"></script>
<script src="src/js/lz-string.min.js"></script> <script src="src/js/lz-string.min.js"></script>
@ -80,6 +80,7 @@
<br><br> <br><br>
<p>May 10th, 2022 - Added Girls &amp; Panzer (the original series).</p> <p>May 10th, 2022 - Added Girls &amp; Panzer (the original series).</p>
<p>October 22nd, 2022 - Added Do it Yourself and Bocchi the Rock!.</p>
</div> </div>
</div> </div>
</body> </body>

View File

@ -1 +0,0 @@
rsync -rtvzP root@kikurage.moe:/var/www/kikurage/ ~/git/kikurage/src/

View File

@ -1 +0,0 @@
rsync -rtvzP ~/git/charasort root@junko.cafe:/var/www/GuP-sorter

View File

@ -1,21 +1,85 @@
from datetime import datetime
import sys
import requests import requests
import re import re
URL = "https://myanimelist.net/anime/14131/Girls___Panzer/characters" # Regex patterns
re_title = '<h1 class="title-name h1_bold_none"><strong>(.*?)</strong></h1>'
re_character = '<img alt=\"(.*)\" width.*characters\/(.*?.jpg)' re_character = '<img alt=\"(.*)\" width.*characters\/(.*?.jpg)'
# Structure to store characters in the dataset
struct_character = r'''name: "{fname}", struct_character = r'''name: "{fname}",
img: "{furl}", img: "{furl}",
opts: {fopt} opts: {fopt}
''' '''
res = requests.get(URL) # Dataset template
characters = re.findall(re_character, res.text) TEMPLATE_SUB = '{{ name: "{fname}", key: "{fkey}" }},\n'
TEMPLATE_DATA_FILE = """
dataSetVersion = "{fdate}"; // Change this when creating a new data set version. YYYY-MM-DD format.
dataSet[dataSetVersion] = {{}};
ret = "" dataSet[dataSetVersion].options = [
for name, url in characters: {{
form = struct_character name: "Filter by Series Entry",
res = "{" + form.format(fname = name, furl = url, fopt = "{}") + "}" key: "series",
ret += res + ",\n" tooltip: "Check this to restrict to certain series.",
checked: false,
sub: [
{fsubs}
]
}},
];
print(ret) dataSet[dataSetVersion].characterData = [
{fdata}
];
"""
def build(url):
res = requests.get(url + "/characters")
title = re.findall(re_title, res.text)[0]
key = "".join(c[0] for c in title.split())
characters = re.findall(re_character, res.text)
sub = TEMPLATE_SUB.format(fname = title, fkey = key)
data = ""
for name, img_url in characters:
form = struct_character
res = "{" + form.format(fname = name,
furl = img_url,
fopt = '{{series : ["{fkey}"] }}'.format(fkey = key)) + "}"
data += res + ",\n"
return data, sub
def write(data, subs):
# Fill in the template
date = datetime.today().strftime('%Y-%m-%d %H:%M:%S')
res = TEMPLATE_DATA_FILE.format(
fdate = date,
fsubs = subs,
fdata = data)
# Write to file
file_name = "data/data.js"
with open(file_name, "w") as f:
f.write(res)
if __name__ == "__main__":
#if len(sys.argv) != 2:
# sys.exit("Usage: python build-dataset.py [url]")
#url = sys.argv[1]
subs = ''
data = ''
with open("series.txt", "r") as f:
series = f.readlines()
for url in series:
dt, sub = build(url)
data += dt
subs += sub
write(data, subs)

View File

@ -1,246 +0,0 @@
dataSetVersion = "2022-05-10"; // Change this when creating a new data set version. YYYY-MM-DD format.
dataSet[dataSetVersion] = {};
dataSet[dataSetVersion].options = [
{
name: "Filter by Series Entry",
key: "series",
tooltip: "Check this to restrict to certain series.",
checked: false,
sub: [
{ name: "Girls & Panzer", tooltip: "Girls & Panzer", key: "GP" },
]
},
];
dataSet[dataSetVersion].characterData = [
{name: "Akiyama, Yukari",
img: "13/220639.jpg",
opts: {}
},
{name: "Isuzu, Hana",
img: "14/183595.jpg",
opts: {}
},
{name: "Nishizumi, Miho",
img: "15/208209.jpg",
opts: {}
},
{name: "Reizei, Mako",
img: "3/253025.jpg",
opts: {}
},
{name: "Takebe, Saori",
img: "9/253031.jpg",
opts: {}
},
{name: "Akaboshi, Koume",
img: "6/264545.jpg",
opts: {}
},
{name: "Akiyama, Jungorou",
img: "10/251349.jpg",
opts: {}
},
{name: "Akiyama, Yoshiko",
img: "3/251319.jpg",
opts: {}
},
{name: "Alisa",
img: "8/186485.jpg",
opts: {}
},
{name: "Anzai, Chiyomi",
img: "5/277937.jpg",
opts: {}
},
{name: "Assam",
img: "3/185595.jpg",
opts: {}
},
{name: "Chouno, Ami",
img: "13/184299.jpg",
opts: {}
},
{name: "Darjeeling",
img: "10/355674.jpg",
opts: {}
},
{name: "Gotou, Moyoko",
img: "12/249083.jpg",
opts: {}
},
{name: "Hoshino",
img: "5/200517.jpg",
opts: {}
},
{name: "Inatomi, Hibiki",
img: "9/266043.jpg",
opts: {}
},
{name: "Isobe, Noriko",
img: "4/306126.jpg",
opts: {}
},
{name: "Isuzu, Yuri",
img: "14/200523.jpg",
opts: {}
},
{name: "Itsumi, Erika",
img: "2/360836.jpg",
opts: {}
},
{name: "Kadotani, Anzu",
img: "9/229625.jpg",
opts: {}
},
{name: "Katyusha",
img: "14/193255.jpg",
opts: {}
},
{name: "Kawanishi, Shinobu",
img: "14/449666.jpg",
opts: {}
},
{name: "Kawashima, Momo",
img: "2/184305.jpg",
opts: {}
},
{name: "Kei",
img: "14/381763.jpg",
opts: {}
},
{name: "Kondou, Taeko",
img: "11/391698.jpg",
opts: {}
},
{name: "Konparu, Nozomi",
img: "13/249081.jpg",
opts: {}
},
{name: "Koyama, Yuzu",
img: "12/191392.jpg",
opts: {}
},
{name: "Maruyama, Saki",
img: "2/184293.jpg",
opts: {}
},
{name: "Matsumoto, Riko",
img: "3/186191.jpg",
opts: {}
},
{name: "Momoga",
img: "14/190906.jpg",
opts: {}
},
{name: "Nakajima, Satoko",
img: "7/200519.jpg",
opts: {}
},
{name: "Naomi",
img: "16/187787.jpg",
opts: {}
},
{name: "Nekota",
img: "8/190904.jpg",
opts: {}
},
{name: "Nilgiri",
img: "3/441097.jpg",
opts: {}
},
{name: "Nishizumi, Maho",
img: "9/200657.jpg",
opts: {}
},
{name: "Nishizumi, Shiho",
img: "2/249079.jpg",
opts: {}
},
{name: "Nogami, Takeko",
img: "12/184467.jpg",
opts: {}
},
{name: "Nonna",
img: "10/197747.jpg",
opts: {}
},
{name: "Oono, Aya",
img: "3/184301.jpg",
opts: {}
},
{name: "Orange Pekoe",
img: "13/185597.jpg",
opts: {}
},
{name: "Ou, Taiga",
img: "6/252081.jpg",
opts: {}
},
{name: "Piyotan",
img: "12/190908.jpg",
opts: {}
},
{name: "Reizei, Hisako",
img: "8/249077.jpg",
opts: {}
},
{name: "Rukuriri",
img: "16/313972.jpg",
opts: {}
},
{name: "Sakaguchi, Karina",
img: "9/184297.jpg",
opts: {}
},
{name: "Sasagawa, Kanon",
img: "3/266041.jpg",
opts: {}
},
{name: "Sasaki, Akebi",
img: "11/251325.jpg",
opts: {}
},
{name: "Sawa, Azusa",
img: "7/187799.jpg",
opts: {}
},
{name: "Shinzaburou",
img: "8/190910.jpg",
opts: {}
},
{name: "Sono, Midoriko",
img: "12/185967.jpg",
opts: {}
},
{name: "Sugiyama, Kiyomi",
img: "14/184577.jpg",
opts: {}
},
{name: "Suzuki",
img: "6/251329.jpg",
opts: {}
},
{name: "Suzuki, Takako",
img: "9/184579.jpg",
opts: {}
},
{name: "Takashima, Remi",
img: "12/266045.jpg",
opts: {}
},
{name: "Tsuchiya",
img: "7/200521.jpg",
opts: {}
},
{name: "Utsugi, Yuuki",
img: "16/184291.jpg",
opts: {}
},
{name: "Yamagou, Ayumi",
img: "11/184295.jpg",
opts: {}
}
];

180
src/js/data/data.js Normal file
View File

@ -0,0 +1,180 @@
dataSetVersion = "2022-10-22 22:17:11"; // Change this when creating a new data set version. YYYY-MM-DD format.
dataSet[dataSetVersion] = {};
dataSet[dataSetVersion].options = [
{
name: "Filter by Series Entry",
key: "series",
tooltip: "Check this to restrict to certain series.",
checked: false,
sub: [
{ name: "Bocchi the Rock!", key: "BtR" },
{ name: "Girls &amp; Panzer", key: "G&P" },
{ name: "Do It Yourself!!", key: "DIY" },
{ name: "Akiba Meido Sensou", key: "AMS" },
{ name: "Pui Pui Molcar", key: "PPM" },
]
},
];
dataSet[dataSetVersion].characterData = [
{name: "Gotou, Hitori",
img: "8/479213.jpg",
opts: {series : ["BtR"] }
},
{name: "Yamada, Ryou",
img: "16/491303.jpg",
opts: {series : ["BtR"] }
},
{name: "Ijichi, Nijika",
img: "16/491305.jpg",
opts: {series : ["BtR"] }
},
{name: "Kita, Ikuyo",
img: "7/479209.jpg",
opts: {series : ["BtR"] }
},
{name: "Ijichi, Seika",
img: "6/491306.jpg",
opts: {series : ["BtR"] }
},
{name: "Akiyama, Yukari",
img: "13/220639.jpg",
opts: {series : ["G&P"] }
},
{name: "Reizei, Mako",
img: "3/253025.jpg",
opts: {series : ["G&P"] }
},
{name: "Nishizumi, Miho",
img: "11/306124.jpg",
opts: {series : ["G&P"] }
},
{name: "Isuzu, Hana",
img: "14/183595.jpg",
opts: {series : ["G&P"] }
},
{name: "Takebe, Saori",
img: "9/253031.jpg",
opts: {series : ["G&P"] }
},
{name: "Anzai, Chiyomi",
img: "5/277937.jpg",
opts: {series : ["G&P"] }
},
{name: "Katyusha",
img: "14/193255.jpg",
opts: {series : ["G&P"] }
},
{name: "Darjeeling",
img: "10/355674.jpg",
opts: {series : ["G&P"] }
},
{name: "Nishizumi, Maho",
img: "9/200657.jpg",
opts: {series : ["G&P"] }
},
{name: "Matsumoto, Riko",
img: "3/186191.jpg",
opts: {series : ["G&P"] }
},
{name: "Yua, Serufu",
img: "6/437457.jpg",
opts: {series : ["DIY"] }
},
{name: "Suride, Miku",
img: "13/437458.jpg",
opts: {series : ["DIY"] }
},
{name: "Juliet Queen Elizabeth VIII",
img: "3/437454.jpg",
opts: {series : ["DIY"] }
},
{name: "Hikage, Takumi",
img: "15/437456.jpg",
opts: {series : ["DIY"] }
},
{name: "Yasaku, Rei",
img: "2/437459.jpg",
opts: {series : ["DIY"] }
},
{name: "Kouki, Kokoro",
img: "2/437455.jpg",
opts: {series : ["DIY"] }
},
{name: "Serufu&amp;#039;s Mother",
img: "9/490019.jpg",
opts: {series : ["DIY"] }
},
{name: "Kurage-san",
img: "9/490052.jpg",
opts: {series : ["DIY"] }
},
{name: "Hoketsu, Haruko",
img: "2/489910.jpg",
opts: {series : ["DIY"] }
},
{name: "Mannen, Ranko",
img: "5/485602.jpg",
opts: {series : ["AMS"] }
},
{name: "Wahira, Nagomi",
img: "3/485601.jpg",
opts: {series : ["AMS"] }
},
{name: "Yumechi",
img: "9/486045.jpg",
opts: {series : ["AMS"] }
},
{name: "Zoya",
img: "11/491188.jpg",
opts: {series : ["AMS"] }
},
{name: "Okachimachi",
img: "3/486050.jpg",
opts: {series : ["AMS"] }
},
{name: "Shiipon",
img: "13/486046.jpg",
opts: {series : ["AMS"] }
},
{name: "Tenchou",
img: "16/486047.jpg",
opts: {series : ["AMS"] }
},
{name: "Debt Collector",
img: "3/490016.jpg",
opts: {series : ["AMS"] }
},
{name: "Bear Ace Maid",
img: "11/491214.jpg",
opts: {series : ["AMS"] }
},
{name: "Potato",
img: "11/470028.jpg",
opts: {series : ["PPM"] }
},
{name: "Teddy",
img: "16/488491.jpg",
opts: {series : ["PPM"] }
},
{name: "Choko",
img: "12/491226.jpg",
opts: {series : ["PPM"] }
},
{name: "Abbey",
img: "4/491227.jpg",
opts: {series : ["PPM"] }
},
{name: "Shiromo",
img: "4/491228.jpg",
opts: {series : ["PPM"] }
},
{name: "Emergency Molcar",
img: "16/432758.jpg",
opts: {series : ["PPM"] }
},
];

View File

@ -1,228 +0,0 @@
{name: "Akiyama, Yukari",
img: "13/220639.jpg",
opts: {}
},
{name: "Isuzu, Hana",
img: "14/183595.jpg",
opts: {}
},
{name: "Nishizumi, Miho",
img: "15/208209.jpg",
opts: {}
},
{name: "Reizei, Mako",
img: "3/253025.jpg",
opts: {}
},
{name: "Takebe, Saori",
img: "9/253031.jpg",
opts: {}
},
{name: "Akaboshi, Koume",
img: "6/264545.jpg",
opts: {}
},
{name: "Akiyama, Jungorou",
img: "10/251349.jpg",
opts: {}
},
{name: "Akiyama, Yoshiko",
img: "3/251319.jpg",
opts: {}
},
{name: "Alisa",
img: "8/186485.jpg",
opts: {}
},
{name: "Anzai, Chiyomi",
img: "5/277937.jpg",
opts: {}
},
{name: "Assam",
img: "3/185595.jpg",
opts: {}
},
{name: "Chouno, Ami",
img: "13/184299.jpg",
opts: {}
},
{name: "Darjeeling",
img: "10/355674.jpg",
opts: {}
},
{name: "Gotou, Moyoko",
img: "12/249083.jpg",
opts: {}
},
{name: "Hoshino",
img: "5/200517.jpg",
opts: {}
},
{name: "Inatomi, Hibiki",
img: "9/266043.jpg",
opts: {}
},
{name: "Isobe, Noriko",
img: "4/306126.jpg",
opts: {}
},
{name: "Isuzu, Yuri",
img: "14/200523.jpg",
opts: {}
},
{name: "Itsumi, Erika",
img: "2/360836.jpg",
opts: {}
},
{name: "Kadotani, Anzu",
img: "9/229625.jpg",
opts: {}
},
{name: "Katyusha",
img: "14/193255.jpg",
opts: {}
},
{name: "Kawanishi, Shinobu",
img: "14/449666.jpg",
opts: {}
},
{name: "Kawashima, Momo",
img: "2/184305.jpg",
opts: {}
},
{name: "Kei",
img: "14/381763.jpg",
opts: {}
},
{name: "Kondou, Taeko",
img: "11/391698.jpg",
opts: {}
},
{name: "Konparu, Nozomi",
img: "13/249081.jpg",
opts: {}
},
{name: "Koyama, Yuzu",
img: "12/191392.jpg",
opts: {}
},
{name: "Maruyama, Saki",
img: "2/184293.jpg",
opts: {}
},
{name: "Matsumoto, Riko",
img: "3/186191.jpg",
opts: {}
},
{name: "Momoga",
img: "14/190906.jpg",
opts: {}
},
{name: "Nakajima, Satoko",
img: "7/200519.jpg",
opts: {}
},
{name: "Naomi",
img: "16/187787.jpg",
opts: {}
},
{name: "Nekota",
img: "8/190904.jpg",
opts: {}
},
{name: "Nilgiri",
img: "3/441097.jpg",
opts: {}
},
{name: "Nishizumi, Maho",
img: "9/200657.jpg",
opts: {}
},
{name: "Nishizumi, Shiho",
img: "2/249079.jpg",
opts: {}
},
{name: "Nogami, Takeko",
img: "12/184467.jpg",
opts: {}
},
{name: "Nonna",
img: "10/197747.jpg",
opts: {}
},
{name: "Oono, Aya",
img: "3/184301.jpg",
opts: {}
},
{name: "Orange Pekoe",
img: "13/185597.jpg",
opts: {}
},
{name: "Ou, Taiga",
img: "6/252081.jpg",
opts: {}
},
{name: "Piyotan",
img: "12/190908.jpg",
opts: {}
},
{name: "Reizei, Hisako",
img: "8/249077.jpg",
opts: {}
},
{name: "Rukuriri",
img: "16/313972.jpg",
opts: {}
},
{name: "Sakaguchi, Karina",
img: "9/184297.jpg",
opts: {}
},
{name: "Sasagawa, Kanon",
img: "3/266041.jpg",
opts: {}
},
{name: "Sasaki, Akebi",
img: "11/251325.jpg",
opts: {}
},
{name: "Sawa, Azusa",
img: "7/187799.jpg",
opts: {}
},
{name: "Shinzaburou",
img: "8/190910.jpg",
opts: {}
},
{name: "Sono, Midoriko",
img: "12/185967.jpg",
opts: {}
},
{name: "Sugiyama, Kiyomi",
img: "14/184577.jpg",
opts: {}
},
{name: "Suzuki",
img: "6/251329.jpg",
opts: {}
},
{name: "Suzuki, Takako",
img: "9/184579.jpg",
opts: {}
},
{name: "Takashima, Remi",
img: "12/266045.jpg",
opts: {}
},
{name: "Tsuchiya",
img: "7/200521.jpg",
opts: {}
},
{name: "Utsugi, Yuuki",
img: "16/184291.jpg",
opts: {}
},
{name: "Yamagou, Ayumi",
img: "11/184295.jpg",
opts: {}
},

5
src/js/series.txt Normal file
View File

@ -0,0 +1,5 @@
https://myanimelist.net/anime/47917/Bocchi_the_Rock
https://myanimelist.net/anime/14131/Girls___Panzer
https://myanimelist.net/anime/48542/Do_It_Yourselfu
https://myanimelist.net/anime/52193/Akiba_Meido_Sensou
https://myanimelist.net/anime/44235/Pui_Pui_Molcar