last push for meetup

diff --git a/program/everything/crawler.db b/program/everything/crawler.db
index b98b7fb..64b3ee1 100644 (file)
--- a/program/everything/crawler.db
+++ b/program/everything/crawler.db
@@ -1,5 +1,5 @@
 (dp0
-S'Paradiso'
+S'paradiso'
 p1
 (dp2
 S'website'
@@ -9,527 +9,845 @@ p4
 sS'name'
 p5
 g1
-sS'titledawg'
+sS'url'
 p6
-(ipydawg
-DAWG
+S'http://www.paradiso.nl/rss.xml'
 p7
-(dp8
-S'q0'
+sS'venue'
+p8
+S'Paradiso'
 p9
-(ipydawg
-DAWGNode
+sS'dloc'
 p10
-(dp11
-S'final'
+S'nee'
+p11
+sS'db'
 p12
-I00
-sS'number'
-p13
-NsS'children'
+(dp13
+S'63662c13105245c8c98a5cc17443268a'
 p14
 (dp15
-S'\x01'
+S'raw'
 p16
+(Vzondag 30 november 2014 21:00 - Catfish & The Bottlemen - Locatie: Bitterzoet
+p17
+V
+p18
+tp19
+sS'results'
+p20
+(dp21
+S'waar1'
+p22
+VBitterzoet
+p23
+sS'datum1'
+p24
+Vzondag 30 november 2014
+p25
+sS'tijd1'
+p26
+V21:00
+p27
+sS'wat1'
+p28
+VCatfish & The Bottlemen
+p29
+sssS'497afcdf3c8fe95e5b63a7fd3483c88e'
+p30
+(dp31
+g16
+(Vdonderdag 4 december 2014 21:30 - She Keeps Bees
+p32
+g18
+tp33
+sg20
+(dp34
+S'tijd0'
+p35
+V21:30
+p36
+sS'wat0'
+p37
+VShe Keeps Bees
+p38
+sS'datum0'
+p39
+Vdonderdag 4 december 2014
+p40
+sssS'bc20fb8295d411fc5dd3b2b1f6cb0f4e'
+p41
+(dp42
+g16
+(Vdonderdag 30 oktober 2014 22:00 - Dondergrondse: hosted by The Daily Indie @ Kelder
+p43
+g18
+tp44
+sg20
+(dp45
+g35
+V22:00
+p46
+sg37
+VDondergrondse: hosted by The Daily Indie @ Kelder
+p47
+sg39
+Vdonderdag 30 oktober 2014
+p48
+sssS'1aae9feccc63dfc41b653af9cebe97ed'
+p49
+(dp50
+g16
+(Vzaterdag 22 november 2014 21:00 - Selda feat. Boom Pam   -   Locatie: Bitterzoet
+p51
+g18
+tp52
+sg20
+(dp53
+g22
+VBitterzoet
+p54
+sg24
+Vzaterdag 22 november 2014
+p55
+sg26
+V21:00
+p56
+sg28
+VSelda feat. Boom Pam  
+p57
+sssS'0e90f944f7d591d6e3c47bff6ba40301'
+p58
+(dp59
+g16
+(Vvrijdag 7 november 2014 20:00 - The Mahones
+p60
+g18
+tp61
+sg20
+(dp62
+g35
+V20:00
+p63
+sg37
+VThe Mahones
+p64
+sg39
+Vvrijdag 7 november 2014
+p65
+sssS'73801ebeb1dfc852b8a441d4534bfd37'
+p66
+(dp67
+g16
+(Vmaandag 24 november 2014 21:00 - Twin Forks - Locatie: Bitterzoet
+p68
+g18
+tp69
+sg20
+(dp70
+g22
+VBitterzoet
+p71
+sg24
+Vmaandag 24 november 2014
+p72
+sg26
+V21:00
+p73
+sg28
+VTwin Forks
+p74
+sssS'6ae02cb00a5deb63cb417a870900d3cc'
+p75
+(dp76
+g16
+(Vvrijdag 31 oktober 2014 20:30 - Rocket Cinema: Night of the Living Dead - Locatie: Paradiso Noord, Tolhuistuin (tuin)
+p77
+VZombie Horror In Tolhuistuin
+p78
+tp79
+sg20
+(dp80
+S'wat0'
+p81
+g78
+sg22
+VParadiso Noord, Tolhuistuin (tuin)
+p82
+sg24
+Vvrijdag 31 oktober 2014
+p83
+sg26
+V20:30
+p84
+sg28
+VRocket Cinema: Night of the Living Dead
+p85
+sssS'fadc5b32d15a55b4bc60f5448a1c5342'
+p86
+(dp87
+g16
+(Vdonderdag 18 december 2014 19:30 - dEUS
+p88
+V\u2018\u2018Selected Songs 1994-2014\u201d
+p89
+tp90
+sg20
+(dp91
+g81
+VdEUS
+p92
+sg35
+V19:30
+p93
+sg39
+Vdonderdag 18 december 2014
+p94
+sssS'60632ff265f05913f6ebe8f6b0bf1995'
+p95
+(dp96
+g16
+(Vdonderdag 9 oktober 2014 23:30 - Dondergrondse:  hosted by Sweet Dreams @ Kelder
+p97
+g18
+tp98
+sg20
+(dp99
+g35
+V23:30
+p100
+sg37
+VDondergrondse:  hosted by Sweet Dreams @ Kelder
+p101
+sg39
+Vdonderdag 9 oktober 2014
+p102
+sssS'ae8b5e28b321c9caaca5a7b56d892670'
+p103
+(dp104
+g16
+(Vzondag 28 september 2014 11:00 - C4C Affordable Vintage & Fashion Fair   -   Locatie: Paradiso Noord, Tolhuistuin
+p105
+VVintage Fashion Markt & V.I.P. Shoppen
+p106
+tp107
+sg20
+(dp108
+g81
+g106
+sg22
+VParadiso Noord, Tolhuistuin
+p109
+sg24
+Vzondag 28 september 2014
+p110
+sg26
+V11:00
+p111
+sg28
+VC4C Affordable Vintage & Fashion Fair  
+p112
+sssS'c09b1ed0e482c4e27bb7c61fae4d0e15'
+p113
+(dp114
+g16
+(Vvrijdag 16 januari 2015 20:30 - Kraftwerk The Catalogue 12345678 in 3-D - Autobahn (1974)
+p115
+g18
+tp116
+sg20
+(dp117
+g35
+V3-D
+p118
+sg37
+VAutobahn (1974)
+p119
+sg39
+Vvrijdag 16 januari 2015 20:30 - Kraftwerk The Catalogue 12345678 in
+p120
+sssS'91e127c55ea69be373514604cd3f0d54'
+p121
+(dp122
+g16
+(Vzondag 8 maart 2015 20:30 - Thanasis Papakonstantinou & band
+p123
+g18
+tp124
+sg20
+(dp125
+g35
+V20:30
+p126
+sg37
+VThanasis Papakonstantinou & band
+p127
+sg39
+Vzondag 8 maart 2015
+p128
+sssS'492d8f49e92850c4b71345cf78b9cd5c'
+p129
+(dp130
+g16
+(Vzaterdag 15 november 2014 21:00 - Kris Berry & Perquisite afscheidsconcert - Locatie: Bitterzoet
+p131
+g18
+tp132
+sg20
+(dp133
+g22
+VBitterzoet
+p134
+sg24
+Vzaterdag 15 november 2014
+p135
+sg26
+V21:00
+p136
+sg28
+VKris Berry & Perquisite afscheidsconcert
+p137
+sssS'03e21261cc548333f352f3aeefb655ca'
+p138
+(dp139
+g16
+(Vwoensdag 29 oktober 2014 19:00 - Sofia Dragt
+p140
+g18
+tp141
+sg20
+(dp142
+g35
+V19:00
+p143
+sg37
+VSofia Dragt
+p144
+sg39
+Vwoensdag 29 oktober 2014
+p145
+sssS'05acd9f7674136b0497d96d1d7331ed2'
+p146
+(dp147
+g16
+(Vdonderdag 23 oktober 2014 23:30 - Noodlanding!
+p148
+VDansnacht, alternatieve hits
+p149
+tp150
+sg20
+(dp151
+g81
+VNoodlanding!
+p152
+sg35
+V23:30
+p153
+sg39
+Vdonderdag 23 oktober 2014
+p154
+sssS'6e517291b28ec790c33819a477869519'
+p155
+(dp156
+g16
+(Vvrijdag 31 oktober 2014 22:00 - Halloween Hairball - Locatie: Paradiso Noord, Tolhuistuin
+p157
+VThe Freakiest Halloween Special Everrrr...
+p158
+tp159
+sg20
+(dp160
+g81
+g158
+sg22
+VParadiso Noord, Tolhuistuin
+p161
+sg24
+Vvrijdag 31 oktober 2014
+p162
+sg26
+V22:00
+p163
+sg28
+VHalloween Hairball
+p164
+sssS'7b6aeecede53f52256d7ba3278b35679'
+p165
+(dp166
+g16
+(Vzaterdag 4 oktober 2014 20:30 - Fixkes - Locatie: Paradiso Noord, Tolhuistuin
+p167
+g18
+tp168
+sg20
+(dp169
+g22
+VParadiso Noord, Tolhuistuin
+p170
+sg24
+Vzaterdag 4 oktober 2014
+p171
+sg26
+V20:30
+p172
+sg28
+VFixkes
+p173
+sssS'aca18d82bffadc7a8756531c0febec9f'
+p174
+(dp175
+g16
+(Vdonderdag 2 oktober 2014 22:00 - Scoop
+p176
+VParadiso's nieuwe Amsterdamse band-avond
+p177
+tp178
+sg20
+(dp179
+g81
+VScoop
+p180
+sg35
+V22:00
+p181
+sg39
+Vdonderdag 2 oktober 2014
+p182
+sssS'62db0423c9274dc47fdc8713e405cf14'
+p183
+(dp184
+g16
+(Vwoensdag 12 november 2014 20:30 - The Drums   -   Locatie: Paradiso Noord, Tolhuistuin
+p185
+VEncyclopedia
+p186
+tp187
+sg20
+(dp188
+g81
+g186
+sg22
+VParadiso Noord, Tolhuistuin
+p189
+sg24
+Vwoensdag 12 november 2014
+p190
+sg26
+V20:30
+p191
+sg28
+VThe Drums  
+p192
+sssS'c0fbf389b316e3f931dd33fb0ec51a43'
+p193
+(dp194
+g16
+(Vdonderdag 20 november 2014 20:30 - Bombay Bicycle Club
+p195
+VIntelligente indie
+p196
+tp197
+sg20
+(dp198
+g81
+VBombay Bicycle Club
+p199
+sg35
+V20:30
+p200
+sg39
+Vdonderdag 20 november 2014
+p201
+ssssS'last_run'
+p202
+F1410338734.431591
+sS'content'
+p203
+(lp204
+(lp205
+S'<span class="uiWebviewHighlight" style="color: white; background-color: rgb(139, 0, 0);">zaterdag 22 november 2014</span> <span class="uiWebviewHighlight" style="color: white; background-color: red;">21:00</span> - <span class="uiWebviewHighlight" style="color: white; background-color: green;">Selda feat. Boom Pam</span>   -   Locatie: <span class="uiWebviewHighlight" style="color: white; background-color: blue;">Bitterzoet</span>'
+p206
+aS''
+p207
+aa(lp208
+S'<span class="uiWebviewHighlight" style="color: white; background-color: rgb(139, 0, 0);">donderdag 30 oktober 2014</span> <span class="uiWebviewHighlight" style="color: white; background-color: red;">22:00</span> - <span class="uiWebviewHighlight" style="color: white; background-color: green;">Dondergrondse: hosted by The Daily Indie @ Kelder</span>'
+p209
+ag207
+aa(lp210
+S'<span class="uiWebviewHighlight" style="color: white; background-color: rgb(139, 0, 0);">donderdag 9 oktober 2014</span> <span class="uiWebviewHighlight" style="color: white; background-color: red;">23:30</span> - <span class="uiWebviewHighlight" style="color: white; background-color: green;">Dondergrondse:  hosted by Sweet Dreams @ Kelder</span>'
+p211
+ag207
+aa(lp212
+S'<span class="uiWebviewHighlight" style="color: white; background-color: rgb(139, 0, 0);">donderdag 23 oktober 2014</span> <span class="uiWebviewHighlight" style="color: white; background-color: red;">23:30</span> - <span class="uiWebviewHighlight" style="color: white; background-color: green;">Noodlanding!</span>'
+p213
+aS'<span class="uiWebviewHighlight" style="color: white; background-color: green;">Dansnacht, alternatieve hits</span>'
+p214
+aa(lp215
+S'<span class="uiWebviewHighlight" style="color: white; background-color: rgb(139, 0, 0);">donderdag 2 oktober 2014</span> <span class="uiWebviewHighlight" style="color: white; background-color: red;">22:00</span> - <span class="uiWebviewHighlight" style="color: white; background-color: green;">Scoop</span>'
+p216
+aS"Paradiso's nieuwe Amsterdamse band-avond"
+p217
+aa(lp218
+S'donderdag 18 december 2014 19:30 - dEUS'
+p219
+aS'\xe2\x80\x98\xe2\x80\x98Selected Songs 1994-2014\xe2\x80\x9d'
+p220
+aa(lp221
+S'donderdag 4 december 2014 21:30 - She Keeps Bees'
+p222
+ag207
+aa(lp223
+S'donderdag 20 november 2014 20:30 - Bombay Bicycle Club'
+p224
+aS'Intelligente indie'
+p225
+aa(lp226
+S'vrijdag 31 oktober 2014 20:30 - Rocket Cinema: Night of the Living Dead - Locatie: Paradiso Noord, Tolhuistuin (tuin)'
+p227
+aS'Zombie Horror In Tolhuistuin'
+p228
+aa(lp229
+S'vrijdag 31 oktober 2014 22:00 - Halloween Hairball - Locatie: Paradiso Noord, Tolhuistuin'
+p230
+aS'The Freakiest Halloween Special Everrrr...'
+p231
+aasS'headers'
+p232
+(lp233
+S'Title'
+p234
+aS'Summary'
+p235
+asS'summarydawg'
+p236
+(ipydawg
+DAWG
+p237
+(dp238
+S'q0'
+p239
 (ipydawg
 DAWGNode
-p17
-(dp18
-g12
+p240
+(dp241
+S'number'
+p242
+NsS'children'
+p243
+(dp244
+S'\x03'
+p245
+(ipydawg
+DAWGNode
+p246
+(dp247
+g242
+Nsg243
+(dp248
+sS'final'
+p249
+I01
+sbssg249
 I00
-sg13
-Nsg14
-(dp19
+sbsS'_numbers_valid'
+p250
+I00
+sS'register'
+p251
+c__builtin__
+set
+p252
+((lp253
+tp254
+Rp255
+sS'wp'
+p256
+g245
+sbsS'titledawg'
+p257
+(ipydawg
+DAWG
+p258
+(dp259
+g239
+(ipydawg
+DAWGNode
+p260
+(dp261
+g242
+Nsg243
+(dp262
+S'\x01'
+p263
+(ipydawg
+DAWGNode
+p264
+(dp265
+g242
+Nsg243
+(dp266
 S' '
-p20
+p267
 (ipydawg
 DAWGNode
-p21
-(dp22
-g12
-I00
-sg13
-Nsg14
-(dp23
+p268
+(dp269
+g242
+Nsg243
+(dp270
 S'\x02'
-p24
+p271
 (ipydawg
 DAWGNode
-p25
-(dp26
-g12
-I00
-sg13
-Nsg14
-(dp27
-g20
+p272
+(dp273
+g242
+Nsg243
+(dp274
+g267
 (ipydawg
 DAWGNode
-p28
-(dp29
-g12
-I00
-sg13
-Nsg14
-(dp30
+p275
+(dp276
+g242
+Nsg243
+(dp277
 S'-'
-p31
+p278
 (ipydawg
 DAWGNode
-p32
-(dp33
-g12
-I00
-sg13
-Nsg14
-(dp34
-g20
+p279
+(dp280
+g242
+Nsg243
+(dp281
+g267
 (ipydawg
 DAWGNode
-p35
-(dp36
-g12
-I00
-sg13
-Nsg14
-(dp37
-S'\x03'
-p38
+p282
+(dp283
+g242
+Nsg243
+(dp284
+g245
 (ipydawg
 DAWGNode
-p39
-(dp40
-g12
-I01
-sg13
-Nsg14
-(dp41
-g20
+p285
+(dp286
+g242
+Nsg243
+(dp287
+g267
 (ipydawg
 DAWGNode
-p42
-(dp43
-g12
-I00
-sg13
-Nsg14
-(dp44
-g20
+p288
+(dp289
+g242
+Nsg243
+(dp290
+g267
 (ipydawg
 DAWGNode
-p45
-(dp46
-g12
-I00
-sg13
-Nsg14
-(dp47
-g20
+p291
+(dp292
+g242
+Nsg243
+(dp293
+g267
 (ipydawg
 DAWGNode
-p48
-(dp49
-g12
-I00
-sg13
-Nsg14
-(dp50
-g31
+p294
+(dp295
+g242
+Nsg243
+(dp296
+g278
 (ipydawg
 DAWGNode
-p51
-(dp52
-g12
-I00
-sg13
-Nsg14
-(dp53
-g20
+p297
+(dp298
+g242
+Nsg243
+(dp299
+g267
 (ipydawg
 DAWGNode
-p54
-(dp55
-g12
-I00
-sg13
-Nsg14
-(dp56
-g20
+p300
+(dp301
+g242
+Nsg243
+(dp302
+g267
 (ipydawg
 DAWGNode
-p57
-(dp58
-g12
-I00
-sg13
-Nsg14
-(dp59
-g20
+p303
+(dp304
+g242
+Nsg243
+(dp305
+g267
 (ipydawg
 DAWGNode
-p60
-(dp61
-g12
-I00
-sg13
-Nsg14
-(dp62
+p306
+(dp307
+g242
+Nsg243
+(dp308
 S'L'
-p63
+p309
 (ipydawg
 DAWGNode
-p64
-(dp65
-g12
-I00
-sg13
-Nsg14
-(dp66
+p310
+(dp311
+g242
+Nsg243
+(dp312
 S'o'
-p67
+p313
 (ipydawg
 DAWGNode
-p68
-(dp69
-g12
-I00
-sg13
-Nsg14
-(dp70
+p314
+(dp315
+g242
+Nsg243
+(dp316
 S'c'
-p71
+p317
 (ipydawg
 DAWGNode
-p72
-(dp73
-g12
-I00
-sg13
-Nsg14
-(dp74
+p318
+(dp319
+g242
+Nsg243
+(dp320
 S'a'
-p75
+p321
 (ipydawg
 DAWGNode
-p76
-(dp77
-g12
-I00
-sg13
-Nsg14
-(dp78
+p322
+(dp323
+g242
+Nsg243
+(dp324
 S't'
-p79
+p325
 (ipydawg
 DAWGNode
-p80
-(dp81
-g12
-I00
-sg13
-Nsg14
-(dp82
+p326
+(dp327
+g242
+Nsg243
+(dp328
 S'i'
-p83
+p329
 (ipydawg
 DAWGNode
-p84
-(dp85
-g12
-I00
-sg13
-Nsg14
-(dp86
+p330
+(dp331
+g242
+Nsg243
+(dp332
 S'e'
-p87
+p333
 (ipydawg
 DAWGNode
-p88
-(dp89
-g12
-I00
-sg13
-Nsg14
-(dp90
+p334
+(dp335
+g242
+Nsg243
+(dp336
 S':'
-p91
+p337
 (ipydawg
 DAWGNode
-p92
-(dp93
-g12
-I00
-sg13
-Nsg14
-(dp94
-g20
+p338
+(dp339
+g242
+Nsg243
+(dp340
+g267
 (ipydawg
 DAWGNode
-p95
-(dp96
-g12
-I00
-sg13
-Nsg14
-(dp97
+p341
+(dp342
+g242
+Nsg243
+(dp343
 S'\x04'
-p98
+p344
 (ipydawg
 DAWGNode
-p99
-(dp100
-g12
+p345
+(dp346
+g242
+Nsg243
+(dp347
+sg249
 I01
-sg13
-Nsg14
-(dp101
-sbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbsg31
-(ipydawg
-DAWGNode
-p102
-(dp103
-g12
+sbssg249
 I00
-sg13
-Nsg14
-(dp104
-g20
-(ipydawg
-DAWGNode
-p105
-(dp106
-g12
+sbssg249
 I00
-sg13
-Nsg14
-(dp107
-g63
-(ipydawg
-DAWGNode
-p108
-(dp109
-g12
+sbssg249
 I00
-sg13
-Nsg14
-(dp110
-g67
-(ipydawg
-DAWGNode
-p111
-(dp112
-g12
+sbssg249
 I00
-sg13
-Nsg14
-(dp113
-g71
-(ipydawg
-DAWGNode
-p114
-(dp115
-g12
+sbssg249
 I00
-sg13
-Nsg14
-(dp116
-g75
-(ipydawg
-DAWGNode
-p117
-(dp118
-g12
+sbssg249
 I00
-sg13
-Nsg14
-(dp119
-g79
-(ipydawg
-DAWGNode
-p120
-(dp121
-g12
+sbssg249
 I00
-sg13
-Nsg14
-(dp122
-g83
-(ipydawg
-DAWGNode
-p123
-(dp124
-g12
+sbssg249
 I00
-sg13
-Nsg14
-(dp125
-g87
-(ipydawg
-DAWGNode
-p126
-(dp127
-g12
+sbssg249
 I00
-sg13
-Nsg14
-(dp128
-g91
-(ipydawg
-DAWGNode
-p129
-(dp130
-g12
+sbssg249
 I00
-sg13
-Nsg14
-(dp131
-g20
-(ipydawg
-DAWGNode
-p132
-(dp133
-g12
+sbssg249
 I00
-sg13
-Nsg14
-(dp134
-g98
-(ipydawg
-DAWGNode
-p135
-(dp136
-g12
+sbssg249
+I00
+sbssg249
+I00
+sbssg249
+I00
+sbssg249
+I00
+sbssg249
+I00
+sbssg249
 I01
-sg13
-Nsg14
-(dp137
-sbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbsS'_numbers_valid'
-p138
+sbssg249
 I00
-sS'register'
-p139
-c__builtin__
-set
-p140
-((lp141
-g99
-ag60
-ag57
-ag76
-ag84
-ag80
-ag54
-ag45
-ag88
-ag64
-ag92
-ag68
-ag48
-ag95
-ag51
-ag72
-atp142
-Rp143
-sS'wp'
-p144
-S'\x01 \x02 - \x03 - Locatie: \x04'
-p145
-sbsS'dloc'
-p146
-S'Grote zaal'
-p147
-sS'venue'
-p148
-S'Paradiso'
-p149
-sS'content'
-p150
-(lp151
-(lp152
-S'zaterdag 31 mei 2014  - Lentekabinet Festival Afterparty   -   Locatie: Tolhuistuin (zaal)'
-p153
-aS''
-p154
-aa(lp155
-S'<span class="uiWebviewHighlight" style="color: white; background-color: rgb(139, 0, 0);">vrijdag 4 juli 2014</span> <span class="uiWebviewHighlight" style="color: white; background-color: red;">20:30</span> - <span class="uiWebviewHighlight" style="color: white; background-color: green;">The Crimson Projekct</span>   -   Locatie: <span class="uiWebviewHighlight" style="color: white; background-color: blue;">Tolhuistuin (zaal)</span>'
-p156
-aS'Muziek rond King Crimson'
-p157
-aa(lp158
-S'<span class="uiWebviewHighlight" style="color: white; background-color: rgb(139, 0, 0);">dinsdag 10 juni 2014</span> <span class="uiWebviewHighlight" style="color: white; background-color: red;">20:30</span> - <span class="uiWebviewHighlight" style="color: white; background-color: green;">Het Ultieme Natuurkunde Feestje \xe2\x80\x93 keynote Amanda Gefter</span>'
-p159
-ag154
-aa(lp160
-S'<span class="uiWebviewHighlight" style="color: white; background-color: rgb(139, 0, 0);">dinsdag 12 augustus 2014</span> <span class="uiWebviewHighlight" style="color: white; background-color: red;">21:00</span> - <span class="uiWebviewHighlight" style="color: white; background-color: green;">Kevin Drew</span> - Locatie: <span class="uiWebviewHighlight" style="color: white; background-color: blue;">Bitterzoet</span>'
-p161
-aS'mede-oprichter Broken Social Scene solo'
-p162
-aa(lp163
-S'<span class="uiWebviewHighlight" style="color: white; background-color: rgb(139, 0, 0);">vrijdag 4 juli 2014</span> <span class="uiWebviewHighlight" style="color: white; background-color: red;">22:00</span> - <span class="uiWebviewHighlight" style="color: white; background-color: green;">Palenke Soultribe</span>'
-p164
-aS'Electronische muziek en Afro-Colombiaanse ritmes'
-p165
-aa(lp166
-S'<span class="uiWebviewHighlight" style="color: white; background-color: rgb(139, 0, 0);">maandag 3 november 2014</span> <span class="uiWebviewHighlight" style="color: white; background-color: red;">20:15</span> - <span class="uiWebviewHighlight" style="color: white; background-color: green;">Eefje de Visser: Waterwereldsteden</span> - Locatie: <span class="uiWebviewHighlight" style="color: white; background-color: blue;">Het Concertgebouw</span>'
-p167
-ag154
-aa(lp168
-S'zaterdag 27 september 2014 20:30 - A Great Big World - Locatie: Tolhuistuin (zaal)'
-p169
-aS'Hitschrijvers uit New York'
-p170
-aa(lp171
-S'zaterdag 7 juni 2014 23:00 - Benefietavond Marokkaanse Boot'
-p172
-aS'Van Amsterdam naar Tanger'
-p173
-aa(lp174
-S'donderdag 13 november 2014 19:30 - Wouter Hamel'
-p175
-aS'Sprankelende jazzy pop'
-p176
-aa(lp177
-S'vrijdag 13 juni 2014 00:00 - Legends'
-p178
-ag154
-aasS'headers'
-p179
-(lp180
-S'Title'
-p181
-aS'Summary'
-p182
-asS'summarydawg'
-p183
-(ipydawg
-DAWG
-p184
-(dp185
-g9
-(ipydawg
-DAWGNode
-p186
-(dp187
-g12
+sbssg249
 I00
-sg13
-Nsg14
-(dp188
-sbsg138
+sbssg249
+I00
+sbssg249
 I00
-sg139
-g140
-((lp189
-tp190
-Rp191
-sg144
-g154
+sbssg249
+I00
+sbssg249
+I00
+sbssg249
+I00
+sbsg250
+I00
+sg251
+g252
+((lp348
+tp349
+Rp350
+sg256
+S'\x01 \x02 - \x03   -   Locatie: \x04'
+p351
 sbsS'freq'
-p192
-S'1d1u'
-p193
+p352
+S'1w'
+p353
 sS'adress'
-p194
-S'adres'
-p195
+p354
+S'amsterdam'
+p355
 ss.
\ No newline at end of file

diff --git a/program/everything/crawler.py b/program/everything/crawler.py
index dfc70d4..36d84be 100644 (file)
--- a/program/everything/crawler.py
+++ b/program/everything/crawler.py
@@ -1,9 +1,11 @@
 #!/bin/env python
 # -*- coding: utf-8 -*-
 
+import feedparser
+import hashlib
+import os
 import pickle
 import re
-import os
 import sys
 import time
 
@@ -148,7 +150,7 @@ class Crawler():
         if matchs:
             matches = sorted(matchs, key=lambda x: len(x.groups()))
             results['summary'] = list(reversed(matches))[0].groupdict()
-        return results
+        return dict(results['summary'].items() + results['title'].items())
 
     def has_to_run(self, interval, last_run, now):
         time_wait = sum(
@@ -158,14 +160,43 @@ class Crawler():
         return now - last_run >= time_wait
 
     def run_entry(self, name):
-        print 'force run', name
-        pass
+        edict = self.entries[name]
+        if 'db' not in edict:
+            edict['db'] = {}
+        feed = feedparser.parse(edict['url'])
+        for i in feed.entries:
+            hashvalue = hashlib.md5(b'{}{}'.format(
+                i['title'].encode('utf-8'), i['summary'].encode('utf-8')))
+            hashvalue = hashvalue.hexdigest()
+            if hashvalue in edict['db']:
+                continue
+            results = self.test_entry(name, i['title'], i['summary'])
+            edict['db'][hashvalue] = {
+                'results': results,
+                'raw': (i['title'], i['summary'])
+            }
+            print edict['db'][hashvalue]['raw']
+            print edict['db'][hashvalue]['results']
+            print hashvalue
+            raw_input('Press enter for the next one')
 
 
 def main():
     if len(sys.argv) == 5 and sys.argv[1] == 'test':
         cr = Crawler()
         print cr.test_entry(*sys.argv[2:])
+    elif len(sys.argv) == 3 and sys.argv[1] == 'del':
+        cr = Crawler()
+        if sys.argv[2] in cr.entries:
+            del(cr.entries[sys.argv[2]])
+            print 'Succesfull'
+            cr.write()
+        else:
+            print '{} not in the entries'.format(sys.argv[2])
+    elif len(sys.argv) == 3 and sys.argv[1] == 'export':
+        cr = Crawler()
+        for k, v in cr.entries.iteritems():
+            print k, ': ', v
     elif len(sys.argv) == 5 and sys.argv[1] == 'edit':
         cr = Crawler()
         name, key, value = sys.argv[2:]
@@ -193,14 +224,18 @@ def main():
                 cr.run_entry(name)
             else:
                 print 'Skipping because last run was within interval'
+        cr.write()
     elif len(sys.argv) == 2 and sys.argv[1] == 'list':
         cr = Crawler()
         print cr.list_names()
     else:
-        print ('{0} list'
-               '{0} test crawlername title summary'
-               '{0} edit crawlername key value'
-               '{0} run -f {item1 item2 ...|all}').format(sys.argv[0])
+        print ('Usage:\n'
+               '\t{0} del crawlername\n'
+               '\t{0} edit crawlername key value\n'
+               '\t{0} export FILE\n'
+               '\t{0} list\n'
+               '\t{0} run -f {{item1 item2 ...|all}}\n'
+               '\t{0} test crawlername title summary\n').format(sys.argv[0])
 
 if __name__ == '__main__':
     main()

diff --git a/program/everything/input_app.py b/program/everything/input_app.py
index c1cd6e6..8c153b9 100644 (file)
--- a/program/everything/input_app.py
+++ b/program/everything/input_app.py
@@ -25,11 +25,11 @@ def req_pre_pos(req):
         f.write(str(args))
 
 
-def req_pre(req):
+def req_pre(req, args):
     req.log_error('handler')
     req.content_type = 'text/html'
     req.send_http_header()
-    req.write(
+    data = (
         '<html>\n<head>\n'
         '\t<title>HyperFrontend RSS feed input</title>\n'
         '\t<script src="contextmenu_o.js"></script>\n</head>\n<body>\n'
@@ -44,6 +44,9 @@ def req_pre(req):
         '\t\t\t<input type="text" name="adress" class="indat"></td></tr>\n'
         '\t\t<tr><td>Website: </td><td>\n'
         '\t\t\t<input type="text" name="website" class="indat"></td></tr>\n'
+        '\t\t<tr><td>RSS URL: </td><td>\n'
+        '\t\t\t<input type="text" name="url" class="indat"'
+        'value="{}"></td></tr>\n'
         '\t</table>\n'
         '\tSelecteer iets en klik de link<br />\n'
         '\t<button style="color:blue" onclick="javascript:f_waar()">'
@@ -53,7 +56,8 @@ def req_pre(req):
         '\t<button style="color:red" onclick="javascript:f_wann_tijd()">'
         'Tijd</button>\n'
         '\t<button style="color:darkred" onclick="javascript:f_wann_dat()">'
-        'Datum</button>\n')
+        'Datum</button>\n').format(args['url'])
+    req.write(data)
 
 
 def req_post(req):
@@ -82,8 +86,8 @@ def handler(req):
     if req.method == "POST":
         req_pre_pos(req)
     else:
-        req_pre(req)
         args = util.FieldStorage(req)
+        req_pre(req, args)
         if 'url' not in args and 'name' not in args:
             req.write('Something went wrong, empty fields?<br />')
             req.write('<a href="index.html">back</a>')