From eb8a703accd227e9dae237970ec978d4238cf894 Mon Sep 17 00:00:00 2001 From: Mart Lubbers Date: Wed, 10 Sep 2014 10:59:26 +0200 Subject: [PATCH] last push for meetup --- program/everything/crawler.db | 1192 ++++++++++++++++++++----------- program/everything/crawler.py | 51 +- program/everything/input_app.py | 12 +- 3 files changed, 806 insertions(+), 449 deletions(-) diff --git a/program/everything/crawler.db b/program/everything/crawler.db index b98b7fb..64b3ee1 100644 --- a/program/everything/crawler.db +++ b/program/everything/crawler.db @@ -1,5 +1,5 @@ (dp0 -S'Paradiso' +S'paradiso' p1 (dp2 S'website' @@ -9,527 +9,845 @@ p4 sS'name' p5 g1 -sS'titledawg' +sS'url' p6 -(ipydawg -DAWG +S'http://www.paradiso.nl/rss.xml' p7 -(dp8 -S'q0' +sS'venue' +p8 +S'Paradiso' p9 -(ipydawg -DAWGNode +sS'dloc' p10 -(dp11 -S'final' +S'nee' +p11 +sS'db' p12 -I00 -sS'number' -p13 -NsS'children' +(dp13 +S'63662c13105245c8c98a5cc17443268a' p14 (dp15 -S'\x01' +S'raw' p16 +(Vzondag 30 november 2014 21:00 - Catfish & The Bottlemen - Locatie: Bitterzoet +p17 +V +p18 +tp19 +sS'results' +p20 +(dp21 +S'waar1' +p22 +VBitterzoet +p23 +sS'datum1' +p24 +Vzondag 30 november 2014 +p25 +sS'tijd1' +p26 +V21:00 +p27 +sS'wat1' +p28 +VCatfish & The Bottlemen +p29 +sssS'497afcdf3c8fe95e5b63a7fd3483c88e' +p30 +(dp31 +g16 +(Vdonderdag 4 december 2014 21:30 - She Keeps Bees +p32 +g18 +tp33 +sg20 +(dp34 +S'tijd0' +p35 +V21:30 +p36 +sS'wat0' +p37 +VShe Keeps Bees +p38 +sS'datum0' +p39 +Vdonderdag 4 december 2014 +p40 +sssS'bc20fb8295d411fc5dd3b2b1f6cb0f4e' +p41 +(dp42 +g16 +(Vdonderdag 30 oktober 2014 22:00 - Dondergrondse: hosted by The Daily Indie @ Kelder +p43 +g18 +tp44 +sg20 +(dp45 +g35 +V22:00 +p46 +sg37 +VDondergrondse: hosted by The Daily Indie @ Kelder +p47 +sg39 +Vdonderdag 30 oktober 2014 +p48 +sssS'1aae9feccc63dfc41b653af9cebe97ed' +p49 +(dp50 +g16 +(Vzaterdag 22 november 2014 21:00 - Selda feat. Boom Pam - Locatie: Bitterzoet +p51 +g18 +tp52 +sg20 +(dp53 +g22 +VBitterzoet +p54 +sg24 +Vzaterdag 22 november 2014 +p55 +sg26 +V21:00 +p56 +sg28 +VSelda feat. Boom Pam +p57 +sssS'0e90f944f7d591d6e3c47bff6ba40301' +p58 +(dp59 +g16 +(Vvrijdag 7 november 2014 20:00 - The Mahones +p60 +g18 +tp61 +sg20 +(dp62 +g35 +V20:00 +p63 +sg37 +VThe Mahones +p64 +sg39 +Vvrijdag 7 november 2014 +p65 +sssS'73801ebeb1dfc852b8a441d4534bfd37' +p66 +(dp67 +g16 +(Vmaandag 24 november 2014 21:00 - Twin Forks - Locatie: Bitterzoet +p68 +g18 +tp69 +sg20 +(dp70 +g22 +VBitterzoet +p71 +sg24 +Vmaandag 24 november 2014 +p72 +sg26 +V21:00 +p73 +sg28 +VTwin Forks +p74 +sssS'6ae02cb00a5deb63cb417a870900d3cc' +p75 +(dp76 +g16 +(Vvrijdag 31 oktober 2014 20:30 - Rocket Cinema: Night of the Living Dead - Locatie: Paradiso Noord, Tolhuistuin (tuin) +p77 +VZombie Horror In Tolhuistuin +p78 +tp79 +sg20 +(dp80 +S'wat0' +p81 +g78 +sg22 +VParadiso Noord, Tolhuistuin (tuin) +p82 +sg24 +Vvrijdag 31 oktober 2014 +p83 +sg26 +V20:30 +p84 +sg28 +VRocket Cinema: Night of the Living Dead +p85 +sssS'fadc5b32d15a55b4bc60f5448a1c5342' +p86 +(dp87 +g16 +(Vdonderdag 18 december 2014 19:30 - dEUS +p88 +V\u2018\u2018Selected Songs 1994-2014\u201d +p89 +tp90 +sg20 +(dp91 +g81 +VdEUS +p92 +sg35 +V19:30 +p93 +sg39 +Vdonderdag 18 december 2014 +p94 +sssS'60632ff265f05913f6ebe8f6b0bf1995' +p95 +(dp96 +g16 +(Vdonderdag 9 oktober 2014 23:30 - Dondergrondse: hosted by Sweet Dreams @ Kelder +p97 +g18 +tp98 +sg20 +(dp99 +g35 +V23:30 +p100 +sg37 +VDondergrondse: hosted by Sweet Dreams @ Kelder +p101 +sg39 +Vdonderdag 9 oktober 2014 +p102 +sssS'ae8b5e28b321c9caaca5a7b56d892670' +p103 +(dp104 +g16 +(Vzondag 28 september 2014 11:00 - C4C Affordable Vintage & Fashion Fair - Locatie: Paradiso Noord, Tolhuistuin +p105 +VVintage Fashion Markt & V.I.P. Shoppen +p106 +tp107 +sg20 +(dp108 +g81 +g106 +sg22 +VParadiso Noord, Tolhuistuin +p109 +sg24 +Vzondag 28 september 2014 +p110 +sg26 +V11:00 +p111 +sg28 +VC4C Affordable Vintage & Fashion Fair +p112 +sssS'c09b1ed0e482c4e27bb7c61fae4d0e15' +p113 +(dp114 +g16 +(Vvrijdag 16 januari 2015 20:30 - Kraftwerk The Catalogue 12345678 in 3-D - Autobahn (1974) +p115 +g18 +tp116 +sg20 +(dp117 +g35 +V3-D +p118 +sg37 +VAutobahn (1974) +p119 +sg39 +Vvrijdag 16 januari 2015 20:30 - Kraftwerk The Catalogue 12345678 in +p120 +sssS'91e127c55ea69be373514604cd3f0d54' +p121 +(dp122 +g16 +(Vzondag 8 maart 2015 20:30 - Thanasis Papakonstantinou & band +p123 +g18 +tp124 +sg20 +(dp125 +g35 +V20:30 +p126 +sg37 +VThanasis Papakonstantinou & band +p127 +sg39 +Vzondag 8 maart 2015 +p128 +sssS'492d8f49e92850c4b71345cf78b9cd5c' +p129 +(dp130 +g16 +(Vzaterdag 15 november 2014 21:00 - Kris Berry & Perquisite afscheidsconcert - Locatie: Bitterzoet +p131 +g18 +tp132 +sg20 +(dp133 +g22 +VBitterzoet +p134 +sg24 +Vzaterdag 15 november 2014 +p135 +sg26 +V21:00 +p136 +sg28 +VKris Berry & Perquisite afscheidsconcert +p137 +sssS'03e21261cc548333f352f3aeefb655ca' +p138 +(dp139 +g16 +(Vwoensdag 29 oktober 2014 19:00 - Sofia Dragt +p140 +g18 +tp141 +sg20 +(dp142 +g35 +V19:00 +p143 +sg37 +VSofia Dragt +p144 +sg39 +Vwoensdag 29 oktober 2014 +p145 +sssS'05acd9f7674136b0497d96d1d7331ed2' +p146 +(dp147 +g16 +(Vdonderdag 23 oktober 2014 23:30 - Noodlanding! +p148 +VDansnacht, alternatieve hits +p149 +tp150 +sg20 +(dp151 +g81 +VNoodlanding! +p152 +sg35 +V23:30 +p153 +sg39 +Vdonderdag 23 oktober 2014 +p154 +sssS'6e517291b28ec790c33819a477869519' +p155 +(dp156 +g16 +(Vvrijdag 31 oktober 2014 22:00 - Halloween Hairball - Locatie: Paradiso Noord, Tolhuistuin +p157 +VThe Freakiest Halloween Special Everrrr... +p158 +tp159 +sg20 +(dp160 +g81 +g158 +sg22 +VParadiso Noord, Tolhuistuin +p161 +sg24 +Vvrijdag 31 oktober 2014 +p162 +sg26 +V22:00 +p163 +sg28 +VHalloween Hairball +p164 +sssS'7b6aeecede53f52256d7ba3278b35679' +p165 +(dp166 +g16 +(Vzaterdag 4 oktober 2014 20:30 - Fixkes - Locatie: Paradiso Noord, Tolhuistuin +p167 +g18 +tp168 +sg20 +(dp169 +g22 +VParadiso Noord, Tolhuistuin +p170 +sg24 +Vzaterdag 4 oktober 2014 +p171 +sg26 +V20:30 +p172 +sg28 +VFixkes +p173 +sssS'aca18d82bffadc7a8756531c0febec9f' +p174 +(dp175 +g16 +(Vdonderdag 2 oktober 2014 22:00 - Scoop +p176 +VParadiso's nieuwe Amsterdamse band-avond +p177 +tp178 +sg20 +(dp179 +g81 +VScoop +p180 +sg35 +V22:00 +p181 +sg39 +Vdonderdag 2 oktober 2014 +p182 +sssS'62db0423c9274dc47fdc8713e405cf14' +p183 +(dp184 +g16 +(Vwoensdag 12 november 2014 20:30 - The Drums - Locatie: Paradiso Noord, Tolhuistuin +p185 +VEncyclopedia +p186 +tp187 +sg20 +(dp188 +g81 +g186 +sg22 +VParadiso Noord, Tolhuistuin +p189 +sg24 +Vwoensdag 12 november 2014 +p190 +sg26 +V20:30 +p191 +sg28 +VThe Drums +p192 +sssS'c0fbf389b316e3f931dd33fb0ec51a43' +p193 +(dp194 +g16 +(Vdonderdag 20 november 2014 20:30 - Bombay Bicycle Club +p195 +VIntelligente indie +p196 +tp197 +sg20 +(dp198 +g81 +VBombay Bicycle Club +p199 +sg35 +V20:30 +p200 +sg39 +Vdonderdag 20 november 2014 +p201 +ssssS'last_run' +p202 +F1410338734.431591 +sS'content' +p203 +(lp204 +(lp205 +S'zaterdag 22 november 2014 21:00 - Selda feat. Boom Pam - Locatie: Bitterzoet' +p206 +aS'' +p207 +aa(lp208 +S'donderdag 30 oktober 2014 22:00 - Dondergrondse: hosted by The Daily Indie @ Kelder' +p209 +ag207 +aa(lp210 +S'donderdag 9 oktober 2014 23:30 - Dondergrondse: hosted by Sweet Dreams @ Kelder' +p211 +ag207 +aa(lp212 +S'donderdag 23 oktober 2014 23:30 - Noodlanding!' +p213 +aS'Dansnacht, alternatieve hits' +p214 +aa(lp215 +S'donderdag 2 oktober 2014 22:00 - Scoop' +p216 +aS"Paradiso's nieuwe Amsterdamse band-avond" +p217 +aa(lp218 +S'donderdag 18 december 2014 19:30 - dEUS' +p219 +aS'\xe2\x80\x98\xe2\x80\x98Selected Songs 1994-2014\xe2\x80\x9d' +p220 +aa(lp221 +S'donderdag 4 december 2014 21:30 - She Keeps Bees' +p222 +ag207 +aa(lp223 +S'donderdag 20 november 2014 20:30 - Bombay Bicycle Club' +p224 +aS'Intelligente indie' +p225 +aa(lp226 +S'vrijdag 31 oktober 2014 20:30 - Rocket Cinema: Night of the Living Dead - Locatie: Paradiso Noord, Tolhuistuin (tuin)' +p227 +aS'Zombie Horror In Tolhuistuin' +p228 +aa(lp229 +S'vrijdag 31 oktober 2014 22:00 - Halloween Hairball - Locatie: Paradiso Noord, Tolhuistuin' +p230 +aS'The Freakiest Halloween Special Everrrr...' +p231 +aasS'headers' +p232 +(lp233 +S'Title' +p234 +aS'Summary' +p235 +asS'summarydawg' +p236 +(ipydawg +DAWG +p237 +(dp238 +S'q0' +p239 (ipydawg DAWGNode -p17 -(dp18 -g12 +p240 +(dp241 +S'number' +p242 +NsS'children' +p243 +(dp244 +S'\x03' +p245 +(ipydawg +DAWGNode +p246 +(dp247 +g242 +Nsg243 +(dp248 +sS'final' +p249 +I01 +sbssg249 I00 -sg13 -Nsg14 -(dp19 +sbsS'_numbers_valid' +p250 +I00 +sS'register' +p251 +c__builtin__ +set +p252 +((lp253 +tp254 +Rp255 +sS'wp' +p256 +g245 +sbsS'titledawg' +p257 +(ipydawg +DAWG +p258 +(dp259 +g239 +(ipydawg +DAWGNode +p260 +(dp261 +g242 +Nsg243 +(dp262 +S'\x01' +p263 +(ipydawg +DAWGNode +p264 +(dp265 +g242 +Nsg243 +(dp266 S' ' -p20 +p267 (ipydawg DAWGNode -p21 -(dp22 -g12 -I00 -sg13 -Nsg14 -(dp23 +p268 +(dp269 +g242 +Nsg243 +(dp270 S'\x02' -p24 +p271 (ipydawg DAWGNode -p25 -(dp26 -g12 -I00 -sg13 -Nsg14 -(dp27 -g20 +p272 +(dp273 +g242 +Nsg243 +(dp274 +g267 (ipydawg DAWGNode -p28 -(dp29 -g12 -I00 -sg13 -Nsg14 -(dp30 +p275 +(dp276 +g242 +Nsg243 +(dp277 S'-' -p31 +p278 (ipydawg DAWGNode -p32 -(dp33 -g12 -I00 -sg13 -Nsg14 -(dp34 -g20 +p279 +(dp280 +g242 +Nsg243 +(dp281 +g267 (ipydawg DAWGNode -p35 -(dp36 -g12 -I00 -sg13 -Nsg14 -(dp37 -S'\x03' -p38 +p282 +(dp283 +g242 +Nsg243 +(dp284 +g245 (ipydawg DAWGNode -p39 -(dp40 -g12 -I01 -sg13 -Nsg14 -(dp41 -g20 +p285 +(dp286 +g242 +Nsg243 +(dp287 +g267 (ipydawg DAWGNode -p42 -(dp43 -g12 -I00 -sg13 -Nsg14 -(dp44 -g20 +p288 +(dp289 +g242 +Nsg243 +(dp290 +g267 (ipydawg DAWGNode -p45 -(dp46 -g12 -I00 -sg13 -Nsg14 -(dp47 -g20 +p291 +(dp292 +g242 +Nsg243 +(dp293 +g267 (ipydawg DAWGNode -p48 -(dp49 -g12 -I00 -sg13 -Nsg14 -(dp50 -g31 +p294 +(dp295 +g242 +Nsg243 +(dp296 +g278 (ipydawg DAWGNode -p51 -(dp52 -g12 -I00 -sg13 -Nsg14 -(dp53 -g20 +p297 +(dp298 +g242 +Nsg243 +(dp299 +g267 (ipydawg DAWGNode -p54 -(dp55 -g12 -I00 -sg13 -Nsg14 -(dp56 -g20 +p300 +(dp301 +g242 +Nsg243 +(dp302 +g267 (ipydawg DAWGNode -p57 -(dp58 -g12 -I00 -sg13 -Nsg14 -(dp59 -g20 +p303 +(dp304 +g242 +Nsg243 +(dp305 +g267 (ipydawg DAWGNode -p60 -(dp61 -g12 -I00 -sg13 -Nsg14 -(dp62 +p306 +(dp307 +g242 +Nsg243 +(dp308 S'L' -p63 +p309 (ipydawg DAWGNode -p64 -(dp65 -g12 -I00 -sg13 -Nsg14 -(dp66 +p310 +(dp311 +g242 +Nsg243 +(dp312 S'o' -p67 +p313 (ipydawg DAWGNode -p68 -(dp69 -g12 -I00 -sg13 -Nsg14 -(dp70 +p314 +(dp315 +g242 +Nsg243 +(dp316 S'c' -p71 +p317 (ipydawg DAWGNode -p72 -(dp73 -g12 -I00 -sg13 -Nsg14 -(dp74 +p318 +(dp319 +g242 +Nsg243 +(dp320 S'a' -p75 +p321 (ipydawg DAWGNode -p76 -(dp77 -g12 -I00 -sg13 -Nsg14 -(dp78 +p322 +(dp323 +g242 +Nsg243 +(dp324 S't' -p79 +p325 (ipydawg DAWGNode -p80 -(dp81 -g12 -I00 -sg13 -Nsg14 -(dp82 +p326 +(dp327 +g242 +Nsg243 +(dp328 S'i' -p83 +p329 (ipydawg DAWGNode -p84 -(dp85 -g12 -I00 -sg13 -Nsg14 -(dp86 +p330 +(dp331 +g242 +Nsg243 +(dp332 S'e' -p87 +p333 (ipydawg DAWGNode -p88 -(dp89 -g12 -I00 -sg13 -Nsg14 -(dp90 +p334 +(dp335 +g242 +Nsg243 +(dp336 S':' -p91 +p337 (ipydawg DAWGNode -p92 -(dp93 -g12 -I00 -sg13 -Nsg14 -(dp94 -g20 +p338 +(dp339 +g242 +Nsg243 +(dp340 +g267 (ipydawg DAWGNode -p95 -(dp96 -g12 -I00 -sg13 -Nsg14 -(dp97 +p341 +(dp342 +g242 +Nsg243 +(dp343 S'\x04' -p98 +p344 (ipydawg DAWGNode -p99 -(dp100 -g12 +p345 +(dp346 +g242 +Nsg243 +(dp347 +sg249 I01 -sg13 -Nsg14 -(dp101 -sbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbsg31 -(ipydawg -DAWGNode -p102 -(dp103 -g12 +sbssg249 I00 -sg13 -Nsg14 -(dp104 -g20 -(ipydawg -DAWGNode -p105 -(dp106 -g12 +sbssg249 I00 -sg13 -Nsg14 -(dp107 -g63 -(ipydawg -DAWGNode -p108 -(dp109 -g12 +sbssg249 I00 -sg13 -Nsg14 -(dp110 -g67 -(ipydawg -DAWGNode -p111 -(dp112 -g12 +sbssg249 I00 -sg13 -Nsg14 -(dp113 -g71 -(ipydawg -DAWGNode -p114 -(dp115 -g12 +sbssg249 I00 -sg13 -Nsg14 -(dp116 -g75 -(ipydawg -DAWGNode -p117 -(dp118 -g12 +sbssg249 I00 -sg13 -Nsg14 -(dp119 -g79 -(ipydawg -DAWGNode -p120 -(dp121 -g12 +sbssg249 I00 -sg13 -Nsg14 -(dp122 -g83 -(ipydawg -DAWGNode -p123 -(dp124 -g12 +sbssg249 I00 -sg13 -Nsg14 -(dp125 -g87 -(ipydawg -DAWGNode -p126 -(dp127 -g12 +sbssg249 I00 -sg13 -Nsg14 -(dp128 -g91 -(ipydawg -DAWGNode -p129 -(dp130 -g12 +sbssg249 I00 -sg13 -Nsg14 -(dp131 -g20 -(ipydawg -DAWGNode -p132 -(dp133 -g12 +sbssg249 I00 -sg13 -Nsg14 -(dp134 -g98 -(ipydawg -DAWGNode -p135 -(dp136 -g12 +sbssg249 +I00 +sbssg249 +I00 +sbssg249 +I00 +sbssg249 +I00 +sbssg249 +I00 +sbssg249 I01 -sg13 -Nsg14 -(dp137 -sbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbsS'_numbers_valid' -p138 +sbssg249 I00 -sS'register' -p139 -c__builtin__ -set -p140 -((lp141 -g99 -ag60 -ag57 -ag76 -ag84 -ag80 -ag54 -ag45 -ag88 -ag64 -ag92 -ag68 -ag48 -ag95 -ag51 -ag72 -atp142 -Rp143 -sS'wp' -p144 -S'\x01 \x02 - \x03 - Locatie: \x04' -p145 -sbsS'dloc' -p146 -S'Grote zaal' -p147 -sS'venue' -p148 -S'Paradiso' -p149 -sS'content' -p150 -(lp151 -(lp152 -S'zaterdag 31 mei 2014 - Lentekabinet Festival Afterparty - Locatie: Tolhuistuin (zaal)' -p153 -aS'' -p154 -aa(lp155 -S'vrijdag 4 juli 2014 20:30 - The Crimson Projekct - Locatie: Tolhuistuin (zaal)' -p156 -aS'Muziek rond King Crimson' -p157 -aa(lp158 -S'dinsdag 10 juni 2014 20:30 - Het Ultieme Natuurkunde Feestje \xe2\x80\x93 keynote Amanda Gefter' -p159 -ag154 -aa(lp160 -S'dinsdag 12 augustus 2014 21:00 - Kevin Drew - Locatie: Bitterzoet' -p161 -aS'mede-oprichter Broken Social Scene solo' -p162 -aa(lp163 -S'vrijdag 4 juli 2014 22:00 - Palenke Soultribe' -p164 -aS'Electronische muziek en Afro-Colombiaanse ritmes' -p165 -aa(lp166 -S'maandag 3 november 2014 20:15 - Eefje de Visser: Waterwereldsteden - Locatie: Het Concertgebouw' -p167 -ag154 -aa(lp168 -S'zaterdag 27 september 2014 20:30 - A Great Big World - Locatie: Tolhuistuin (zaal)' -p169 -aS'Hitschrijvers uit New York' -p170 -aa(lp171 -S'zaterdag 7 juni 2014 23:00 - Benefietavond Marokkaanse Boot' -p172 -aS'Van Amsterdam naar Tanger' -p173 -aa(lp174 -S'donderdag 13 november 2014 19:30 - Wouter Hamel' -p175 -aS'Sprankelende jazzy pop' -p176 -aa(lp177 -S'vrijdag 13 juni 2014 00:00 - Legends' -p178 -ag154 -aasS'headers' -p179 -(lp180 -S'Title' -p181 -aS'Summary' -p182 -asS'summarydawg' -p183 -(ipydawg -DAWG -p184 -(dp185 -g9 -(ipydawg -DAWGNode -p186 -(dp187 -g12 +sbssg249 I00 -sg13 -Nsg14 -(dp188 -sbsg138 +sbssg249 +I00 +sbssg249 I00 -sg139 -g140 -((lp189 -tp190 -Rp191 -sg144 -g154 +sbssg249 +I00 +sbssg249 +I00 +sbssg249 +I00 +sbsg250 +I00 +sg251 +g252 +((lp348 +tp349 +Rp350 +sg256 +S'\x01 \x02 - \x03 - Locatie: \x04' +p351 sbsS'freq' -p192 -S'1d1u' -p193 +p352 +S'1w' +p353 sS'adress' -p194 -S'adres' -p195 +p354 +S'amsterdam' +p355 ss. \ No newline at end of file diff --git a/program/everything/crawler.py b/program/everything/crawler.py index dfc70d4..36d84be 100644 --- a/program/everything/crawler.py +++ b/program/everything/crawler.py @@ -1,9 +1,11 @@ #!/bin/env python # -*- coding: utf-8 -*- +import feedparser +import hashlib +import os import pickle import re -import os import sys import time @@ -148,7 +150,7 @@ class Crawler(): if matchs: matches = sorted(matchs, key=lambda x: len(x.groups())) results['summary'] = list(reversed(matches))[0].groupdict() - return results + return dict(results['summary'].items() + results['title'].items()) def has_to_run(self, interval, last_run, now): time_wait = sum( @@ -158,14 +160,43 @@ class Crawler(): return now - last_run >= time_wait def run_entry(self, name): - print 'force run', name - pass + edict = self.entries[name] + if 'db' not in edict: + edict['db'] = {} + feed = feedparser.parse(edict['url']) + for i in feed.entries: + hashvalue = hashlib.md5(b'{}{}'.format( + i['title'].encode('utf-8'), i['summary'].encode('utf-8'))) + hashvalue = hashvalue.hexdigest() + if hashvalue in edict['db']: + continue + results = self.test_entry(name, i['title'], i['summary']) + edict['db'][hashvalue] = { + 'results': results, + 'raw': (i['title'], i['summary']) + } + print edict['db'][hashvalue]['raw'] + print edict['db'][hashvalue]['results'] + print hashvalue + raw_input('Press enter for the next one') def main(): if len(sys.argv) == 5 and sys.argv[1] == 'test': cr = Crawler() print cr.test_entry(*sys.argv[2:]) + elif len(sys.argv) == 3 and sys.argv[1] == 'del': + cr = Crawler() + if sys.argv[2] in cr.entries: + del(cr.entries[sys.argv[2]]) + print 'Succesfull' + cr.write() + else: + print '{} not in the entries'.format(sys.argv[2]) + elif len(sys.argv) == 3 and sys.argv[1] == 'export': + cr = Crawler() + for k, v in cr.entries.iteritems(): + print k, ': ', v elif len(sys.argv) == 5 and sys.argv[1] == 'edit': cr = Crawler() name, key, value = sys.argv[2:] @@ -193,14 +224,18 @@ def main(): cr.run_entry(name) else: print 'Skipping because last run was within interval' + cr.write() elif len(sys.argv) == 2 and sys.argv[1] == 'list': cr = Crawler() print cr.list_names() else: - print ('{0} list' - '{0} test crawlername title summary' - '{0} edit crawlername key value' - '{0} run -f {item1 item2 ...|all}').format(sys.argv[0]) + print ('Usage:\n' + '\t{0} del crawlername\n' + '\t{0} edit crawlername key value\n' + '\t{0} export FILE\n' + '\t{0} list\n' + '\t{0} run -f {{item1 item2 ...|all}}\n' + '\t{0} test crawlername title summary\n').format(sys.argv[0]) if __name__ == '__main__': main() diff --git a/program/everything/input_app.py b/program/everything/input_app.py index c1cd6e6..8c153b9 100644 --- a/program/everything/input_app.py +++ b/program/everything/input_app.py @@ -25,11 +25,11 @@ def req_pre_pos(req): f.write(str(args)) -def req_pre(req): +def req_pre(req, args): req.log_error('handler') req.content_type = 'text/html' req.send_http_header() - req.write( + data = ( '\n\n' '\tHyperFrontend RSS feed input\n' '\t\n\n\n' @@ -44,6 +44,9 @@ def req_pre(req): '\t\t\t\n' '\t\tWebsite: \n' '\t\t\t\n' + '\t\tRSS URL: \n' + '\t\t\t\n' '\t\n' '\tSelecteer iets en klik de link
\n' '\t\n' '\t\n') + 'Datum\n').format(args['url']) + req.write(data) def req_post(req): @@ -82,8 +86,8 @@ def handler(req): if req.method == "POST": req_pre_pos(req) else: - req_pre(req) args = util.FieldStorage(req) + req_pre(req, args) if 'url' not in args and 'name' not in args: req.write('Something went wrong, empty fields?
') req.write('back') -- 2.20.1