From 424c2c6efb5e7d98578bf992ee86dbd74e7b18e6 Mon Sep 17 00:00:00 2001 From: Mart Lubbers Date: Sun, 7 Sep 2014 19:44:34 +0200 Subject: [PATCH] last update --- program/everything/crawler.db | 484 ++++++++++++------------ program/everything/crawler.db.backup | 535 +++++++++++++++++++++++++++ program/everything/crawler.py | 6 + 3 files changed, 783 insertions(+), 242 deletions(-) create mode 100644 program/everything/crawler.db.backup diff --git a/program/everything/crawler.db b/program/everything/crawler.db index 74ef366..b98b7fb 100644 --- a/program/everything/crawler.db +++ b/program/everything/crawler.db @@ -21,389 +21,389 @@ p9 DAWGNode p10 (dp11 -S'number' +S'final' p12 -NsS'children' +I00 +sS'number' p13 -(dp14 +NsS'children' +p14 +(dp15 S'\x01' -p15 +p16 (ipydawg DAWGNode -p16 -(dp17 -g12 -Nsg13 +p17 (dp18 +g12 +I00 +sg13 +Nsg14 +(dp19 S' ' -p19 +p20 (ipydawg DAWGNode -p20 -(dp21 -g12 -Nsg13 +p21 (dp22 +g12 +I00 +sg13 +Nsg14 +(dp23 S'\x02' -p23 +p24 (ipydawg DAWGNode -p24 -(dp25 -g12 -Nsg13 +p25 (dp26 -g19 +g12 +I00 +sg13 +Nsg14 +(dp27 +g20 (ipydawg DAWGNode -p27 -(dp28 -g12 -Nsg13 +p28 (dp29 +g12 +I00 +sg13 +Nsg14 +(dp30 S'-' -p30 +p31 (ipydawg DAWGNode -p31 -(dp32 -g12 -Nsg13 +p32 (dp33 -g19 +g12 +I00 +sg13 +Nsg14 +(dp34 +g20 (ipydawg DAWGNode -p34 -(dp35 -g12 -Nsg13 +p35 (dp36 +g12 +I00 +sg13 +Nsg14 +(dp37 S'\x03' -p37 +p38 (ipydawg DAWGNode -p38 -(dp39 -g12 -Nsg13 +p39 (dp40 -g19 +g12 +I01 +sg13 +Nsg14 +(dp41 +g20 (ipydawg DAWGNode -p41 -(dp42 -g12 -Nsg13 +p42 (dp43 -g19 +g12 +I00 +sg13 +Nsg14 +(dp44 +g20 (ipydawg DAWGNode -p44 -(dp45 -g12 -Nsg13 +p45 (dp46 -g19 +g12 +I00 +sg13 +Nsg14 +(dp47 +g20 (ipydawg DAWGNode -p47 -(dp48 -g12 -Nsg13 +p48 (dp49 -g30 +g12 +I00 +sg13 +Nsg14 +(dp50 +g31 (ipydawg DAWGNode -p50 -(dp51 -g12 -Nsg13 +p51 (dp52 -g19 +g12 +I00 +sg13 +Nsg14 +(dp53 +g20 (ipydawg DAWGNode -p53 -(dp54 -g12 -Nsg13 +p54 (dp55 -g19 +g12 +I00 +sg13 +Nsg14 +(dp56 +g20 (ipydawg DAWGNode -p56 -(dp57 -g12 -Nsg13 +p57 (dp58 -g19 +g12 +I00 +sg13 +Nsg14 +(dp59 +g20 (ipydawg DAWGNode -p59 -(dp60 -g12 -Nsg13 +p60 (dp61 +g12 +I00 +sg13 +Nsg14 +(dp62 S'L' -p62 +p63 (ipydawg DAWGNode -p63 -(dp64 -g12 -Nsg13 +p64 (dp65 +g12 +I00 +sg13 +Nsg14 +(dp66 S'o' -p66 +p67 (ipydawg DAWGNode -p67 -(dp68 -g12 -Nsg13 +p68 (dp69 +g12 +I00 +sg13 +Nsg14 +(dp70 S'c' -p70 +p71 (ipydawg DAWGNode -p71 -(dp72 -g12 -Nsg13 +p72 (dp73 +g12 +I00 +sg13 +Nsg14 +(dp74 S'a' -p74 +p75 (ipydawg DAWGNode -p75 -(dp76 -g12 -Nsg13 +p76 (dp77 +g12 +I00 +sg13 +Nsg14 +(dp78 S't' -p78 +p79 (ipydawg DAWGNode -p79 -(dp80 -g12 -Nsg13 +p80 (dp81 +g12 +I00 +sg13 +Nsg14 +(dp82 S'i' -p82 +p83 (ipydawg DAWGNode -p83 -(dp84 -g12 -Nsg13 +p84 (dp85 +g12 +I00 +sg13 +Nsg14 +(dp86 S'e' -p86 +p87 (ipydawg DAWGNode -p87 -(dp88 -g12 -Nsg13 +p88 (dp89 +g12 +I00 +sg13 +Nsg14 +(dp90 S':' -p90 +p91 (ipydawg DAWGNode -p91 -(dp92 -g12 -Nsg13 +p92 (dp93 -g19 +g12 +I00 +sg13 +Nsg14 +(dp94 +g20 (ipydawg DAWGNode -p94 -(dp95 -g12 -Nsg13 +p95 (dp96 +g12 +I00 +sg13 +Nsg14 +(dp97 S'\x04' -p97 +p98 (ipydawg DAWGNode -p98 -(dp99 -g12 -Nsg13 +p99 (dp100 -sS'final' -p101 +g12 I01 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbsg30 +sg13 +Nsg14 +(dp101 +sbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbsg31 (ipydawg DAWGNode p102 (dp103 g12 -Nsg13 +I00 +sg13 +Nsg14 (dp104 -g19 +g20 (ipydawg DAWGNode p105 (dp106 g12 -Nsg13 +I00 +sg13 +Nsg14 (dp107 -g62 +g63 (ipydawg DAWGNode p108 (dp109 g12 -Nsg13 +I00 +sg13 +Nsg14 (dp110 -g66 +g67 (ipydawg DAWGNode p111 (dp112 g12 -Nsg13 +I00 +sg13 +Nsg14 (dp113 -g70 +g71 (ipydawg DAWGNode p114 (dp115 g12 -Nsg13 +I00 +sg13 +Nsg14 (dp116 -g74 +g75 (ipydawg DAWGNode p117 (dp118 g12 -Nsg13 +I00 +sg13 +Nsg14 (dp119 -g78 +g79 (ipydawg DAWGNode p120 (dp121 g12 -Nsg13 +I00 +sg13 +Nsg14 (dp122 -g82 +g83 (ipydawg DAWGNode p123 (dp124 g12 -Nsg13 +I00 +sg13 +Nsg14 (dp125 -g86 +g87 (ipydawg DAWGNode p126 (dp127 g12 -Nsg13 +I00 +sg13 +Nsg14 (dp128 -g90 +g91 (ipydawg DAWGNode p129 (dp130 g12 -Nsg13 +I00 +sg13 +Nsg14 (dp131 -g19 +g20 (ipydawg DAWGNode p132 (dp133 g12 -Nsg13 +I00 +sg13 +Nsg14 (dp134 -g97 +g98 (ipydawg DAWGNode p135 (dp136 g12 -Nsg13 -(dp137 -sg101 -I01 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 I01 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbssg101 -I00 -sbsS'_numbers_valid' +sg13 +Nsg14 +(dp137 +sbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbsS'_numbers_valid' p138 I00 sS'register' @@ -412,22 +412,22 @@ c__builtin__ set p140 ((lp141 -g87 -ag50 -ag91 -ag63 -ag67 -ag94 -ag47 -ag98 -ag71 -ag59 -ag75 -ag53 -ag44 -ag79 -ag83 -ag56 +g99 +ag60 +ag57 +ag76 +ag84 +ag80 +ag54 +ag45 +ag88 +ag64 +ag92 +ag68 +ag48 +ag95 +ag51 +ag72 atp142 Rp143 sS'wp' @@ -511,10 +511,10 @@ DAWGNode p186 (dp187 g12 -Nsg13 -(dp188 -sg101 I00 +sg13 +Nsg14 +(dp188 sbsg138 I00 sg139 @@ -526,7 +526,7 @@ sg144 g154 sbsS'freq' p192 -S'1w' +S'1d1u' p193 sS'adress' p194 diff --git a/program/everything/crawler.db.backup b/program/everything/crawler.db.backup new file mode 100644 index 0000000..74ef366 --- /dev/null +++ b/program/everything/crawler.db.backup @@ -0,0 +1,535 @@ +(dp0 +S'Paradiso' +p1 +(dp2 +S'website' +p3 +S'www.paradiso.nl' +p4 +sS'name' +p5 +g1 +sS'titledawg' +p6 +(ipydawg +DAWG +p7 +(dp8 +S'q0' +p9 +(ipydawg +DAWGNode +p10 +(dp11 +S'number' +p12 +NsS'children' +p13 +(dp14 +S'\x01' +p15 +(ipydawg +DAWGNode +p16 +(dp17 +g12 +Nsg13 +(dp18 +S' ' +p19 +(ipydawg +DAWGNode +p20 +(dp21 +g12 +Nsg13 +(dp22 +S'\x02' +p23 +(ipydawg +DAWGNode +p24 +(dp25 +g12 +Nsg13 +(dp26 +g19 +(ipydawg +DAWGNode +p27 +(dp28 +g12 +Nsg13 +(dp29 +S'-' +p30 +(ipydawg +DAWGNode +p31 +(dp32 +g12 +Nsg13 +(dp33 +g19 +(ipydawg +DAWGNode +p34 +(dp35 +g12 +Nsg13 +(dp36 +S'\x03' +p37 +(ipydawg +DAWGNode +p38 +(dp39 +g12 +Nsg13 +(dp40 +g19 +(ipydawg +DAWGNode +p41 +(dp42 +g12 +Nsg13 +(dp43 +g19 +(ipydawg +DAWGNode +p44 +(dp45 +g12 +Nsg13 +(dp46 +g19 +(ipydawg +DAWGNode +p47 +(dp48 +g12 +Nsg13 +(dp49 +g30 +(ipydawg +DAWGNode +p50 +(dp51 +g12 +Nsg13 +(dp52 +g19 +(ipydawg +DAWGNode +p53 +(dp54 +g12 +Nsg13 +(dp55 +g19 +(ipydawg +DAWGNode +p56 +(dp57 +g12 +Nsg13 +(dp58 +g19 +(ipydawg +DAWGNode +p59 +(dp60 +g12 +Nsg13 +(dp61 +S'L' +p62 +(ipydawg +DAWGNode +p63 +(dp64 +g12 +Nsg13 +(dp65 +S'o' +p66 +(ipydawg +DAWGNode +p67 +(dp68 +g12 +Nsg13 +(dp69 +S'c' +p70 +(ipydawg +DAWGNode +p71 +(dp72 +g12 +Nsg13 +(dp73 +S'a' +p74 +(ipydawg +DAWGNode +p75 +(dp76 +g12 +Nsg13 +(dp77 +S't' +p78 +(ipydawg +DAWGNode +p79 +(dp80 +g12 +Nsg13 +(dp81 +S'i' +p82 +(ipydawg +DAWGNode +p83 +(dp84 +g12 +Nsg13 +(dp85 +S'e' +p86 +(ipydawg +DAWGNode +p87 +(dp88 +g12 +Nsg13 +(dp89 +S':' +p90 +(ipydawg +DAWGNode +p91 +(dp92 +g12 +Nsg13 +(dp93 +g19 +(ipydawg +DAWGNode +p94 +(dp95 +g12 +Nsg13 +(dp96 +S'\x04' +p97 +(ipydawg +DAWGNode +p98 +(dp99 +g12 +Nsg13 +(dp100 +sS'final' +p101 +I01 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbsg30 +(ipydawg +DAWGNode +p102 +(dp103 +g12 +Nsg13 +(dp104 +g19 +(ipydawg +DAWGNode +p105 +(dp106 +g12 +Nsg13 +(dp107 +g62 +(ipydawg +DAWGNode +p108 +(dp109 +g12 +Nsg13 +(dp110 +g66 +(ipydawg +DAWGNode +p111 +(dp112 +g12 +Nsg13 +(dp113 +g70 +(ipydawg +DAWGNode +p114 +(dp115 +g12 +Nsg13 +(dp116 +g74 +(ipydawg +DAWGNode +p117 +(dp118 +g12 +Nsg13 +(dp119 +g78 +(ipydawg +DAWGNode +p120 +(dp121 +g12 +Nsg13 +(dp122 +g82 +(ipydawg +DAWGNode +p123 +(dp124 +g12 +Nsg13 +(dp125 +g86 +(ipydawg +DAWGNode +p126 +(dp127 +g12 +Nsg13 +(dp128 +g90 +(ipydawg +DAWGNode +p129 +(dp130 +g12 +Nsg13 +(dp131 +g19 +(ipydawg +DAWGNode +p132 +(dp133 +g12 +Nsg13 +(dp134 +g97 +(ipydawg +DAWGNode +p135 +(dp136 +g12 +Nsg13 +(dp137 +sg101 +I01 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I01 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbssg101 +I00 +sbsS'_numbers_valid' +p138 +I00 +sS'register' +p139 +c__builtin__ +set +p140 +((lp141 +g87 +ag50 +ag91 +ag63 +ag67 +ag94 +ag47 +ag98 +ag71 +ag59 +ag75 +ag53 +ag44 +ag79 +ag83 +ag56 +atp142 +Rp143 +sS'wp' +p144 +S'\x01 \x02 - \x03 - Locatie: \x04' +p145 +sbsS'dloc' +p146 +S'Grote zaal' +p147 +sS'venue' +p148 +S'Paradiso' +p149 +sS'content' +p150 +(lp151 +(lp152 +S'zaterdag 31 mei 2014 - Lentekabinet Festival Afterparty - Locatie: Tolhuistuin (zaal)' +p153 +aS'' +p154 +aa(lp155 +S'vrijdag 4 juli 2014 20:30 - The Crimson Projekct - Locatie: Tolhuistuin (zaal)' +p156 +aS'Muziek rond King Crimson' +p157 +aa(lp158 +S'dinsdag 10 juni 2014 20:30 - Het Ultieme Natuurkunde Feestje \xe2\x80\x93 keynote Amanda Gefter' +p159 +ag154 +aa(lp160 +S'dinsdag 12 augustus 2014 21:00 - Kevin Drew - Locatie: Bitterzoet' +p161 +aS'mede-oprichter Broken Social Scene solo' +p162 +aa(lp163 +S'vrijdag 4 juli 2014 22:00 - Palenke Soultribe' +p164 +aS'Electronische muziek en Afro-Colombiaanse ritmes' +p165 +aa(lp166 +S'maandag 3 november 2014 20:15 - Eefje de Visser: Waterwereldsteden - Locatie: Het Concertgebouw' +p167 +ag154 +aa(lp168 +S'zaterdag 27 september 2014 20:30 - A Great Big World - Locatie: Tolhuistuin (zaal)' +p169 +aS'Hitschrijvers uit New York' +p170 +aa(lp171 +S'zaterdag 7 juni 2014 23:00 - Benefietavond Marokkaanse Boot' +p172 +aS'Van Amsterdam naar Tanger' +p173 +aa(lp174 +S'donderdag 13 november 2014 19:30 - Wouter Hamel' +p175 +aS'Sprankelende jazzy pop' +p176 +aa(lp177 +S'vrijdag 13 juni 2014 00:00 - Legends' +p178 +ag154 +aasS'headers' +p179 +(lp180 +S'Title' +p181 +aS'Summary' +p182 +asS'summarydawg' +p183 +(ipydawg +DAWG +p184 +(dp185 +g9 +(ipydawg +DAWGNode +p186 +(dp187 +g12 +Nsg13 +(dp188 +sg101 +I00 +sbsg138 +I00 +sg139 +g140 +((lp189 +tp190 +Rp191 +sg144 +g154 +sbsS'freq' +p192 +S'1w' +p193 +sS'adress' +p194 +S'adres' +p195 +ss. \ No newline at end of file diff --git a/program/everything/crawler.py b/program/everything/crawler.py index 8a864b0..dfc70d4 100644 --- a/program/everything/crawler.py +++ b/program/everything/crawler.py @@ -166,6 +166,11 @@ def main(): if len(sys.argv) == 5 and sys.argv[1] == 'test': cr = Crawler() print cr.test_entry(*sys.argv[2:]) + elif len(sys.argv) == 5 and sys.argv[1] == 'edit': + cr = Crawler() + name, key, value = sys.argv[2:] + cr.entries[name][key] = value + cr.write() elif len(sys.argv) >= 2 and sys.argv[1] == 'run': args = sys.argv[2:] force = True if '-f' in args else False @@ -194,6 +199,7 @@ def main(): else: print ('{0} list' '{0} test crawlername title summary' + '{0} edit crawlername key value' '{0} run -f {item1 item2 ...|all}').format(sys.argv[0]) if __name__ == '__main__': -- 2.20.1