last update

diff --git a/program/everything/crawler.db b/program/everything/crawler.db
index 74ef366..b98b7fb 100644 (file)
--- a/program/everything/crawler.db
+++ b/program/everything/crawler.db
@@ -21,389 +21,389 @@ p9
 DAWGNode
 p10
 (dp11
-S'number'
+S'final'
 p12
-NsS'children'
+I00
+sS'number'
 p13
-(dp14
+NsS'children'
+p14
+(dp15
 S'\x01'
-p15
+p16
 (ipydawg
 DAWGNode
-p16
-(dp17
-g12
-Nsg13
+p17
 (dp18
+g12
+I00
+sg13
+Nsg14
+(dp19
 S' '
-p19
+p20
 (ipydawg
 DAWGNode
-p20
-(dp21
-g12
-Nsg13
+p21
 (dp22
+g12
+I00
+sg13
+Nsg14
+(dp23
 S'\x02'
-p23
+p24
 (ipydawg
 DAWGNode
-p24
-(dp25
-g12
-Nsg13
+p25
 (dp26
-g19
+g12
+I00
+sg13
+Nsg14
+(dp27
+g20
 (ipydawg
 DAWGNode
-p27
-(dp28
-g12
-Nsg13
+p28
 (dp29
+g12
+I00
+sg13
+Nsg14
+(dp30
 S'-'
-p30
+p31
 (ipydawg
 DAWGNode
-p31
-(dp32
-g12
-Nsg13
+p32
 (dp33
-g19
+g12
+I00
+sg13
+Nsg14
+(dp34
+g20
 (ipydawg
 DAWGNode
-p34
-(dp35
-g12
-Nsg13
+p35
 (dp36
+g12
+I00
+sg13
+Nsg14
+(dp37
 S'\x03'
-p37
+p38
 (ipydawg
 DAWGNode
-p38
-(dp39
-g12
-Nsg13
+p39
 (dp40
-g19
+g12
+I01
+sg13
+Nsg14
+(dp41
+g20
 (ipydawg
 DAWGNode
-p41
-(dp42
-g12
-Nsg13
+p42
 (dp43
-g19
+g12
+I00
+sg13
+Nsg14
+(dp44
+g20
 (ipydawg
 DAWGNode
-p44
-(dp45
-g12
-Nsg13
+p45
 (dp46
-g19
+g12
+I00
+sg13
+Nsg14
+(dp47
+g20
 (ipydawg
 DAWGNode
-p47
-(dp48
-g12
-Nsg13
+p48
 (dp49
-g30
+g12
+I00
+sg13
+Nsg14
+(dp50
+g31
 (ipydawg
 DAWGNode
-p50
-(dp51
-g12
-Nsg13
+p51
 (dp52
-g19
+g12
+I00
+sg13
+Nsg14
+(dp53
+g20
 (ipydawg
 DAWGNode
-p53
-(dp54
-g12
-Nsg13
+p54
 (dp55
-g19
+g12
+I00
+sg13
+Nsg14
+(dp56
+g20
 (ipydawg
 DAWGNode
-p56
-(dp57
-g12
-Nsg13
+p57
 (dp58
-g19
+g12
+I00
+sg13
+Nsg14
+(dp59
+g20
 (ipydawg
 DAWGNode
-p59
-(dp60
-g12
-Nsg13
+p60
 (dp61
+g12
+I00
+sg13
+Nsg14
+(dp62
 S'L'
-p62
+p63
 (ipydawg
 DAWGNode
-p63
-(dp64
-g12
-Nsg13
+p64
 (dp65
+g12
+I00
+sg13
+Nsg14
+(dp66
 S'o'
-p66
+p67
 (ipydawg
 DAWGNode
-p67
-(dp68
-g12
-Nsg13
+p68
 (dp69
+g12
+I00
+sg13
+Nsg14
+(dp70
 S'c'
-p70
+p71
 (ipydawg
 DAWGNode
-p71
-(dp72
-g12
-Nsg13
+p72
 (dp73
+g12
+I00
+sg13
+Nsg14
+(dp74
 S'a'
-p74
+p75
 (ipydawg
 DAWGNode
-p75
-(dp76
-g12
-Nsg13
+p76
 (dp77
+g12
+I00
+sg13
+Nsg14
+(dp78
 S't'
-p78
+p79
 (ipydawg
 DAWGNode
-p79
-(dp80
-g12
-Nsg13
+p80
 (dp81
+g12
+I00
+sg13
+Nsg14
+(dp82
 S'i'
-p82
+p83
 (ipydawg
 DAWGNode
-p83
-(dp84
-g12
-Nsg13
+p84
 (dp85
+g12
+I00
+sg13
+Nsg14
+(dp86
 S'e'
-p86
+p87
 (ipydawg
 DAWGNode
-p87
-(dp88
-g12
-Nsg13
+p88
 (dp89
+g12
+I00
+sg13
+Nsg14
+(dp90
 S':'
-p90
+p91
 (ipydawg
 DAWGNode
-p91
-(dp92
-g12
-Nsg13
+p92
 (dp93
-g19
+g12
+I00
+sg13
+Nsg14
+(dp94
+g20
 (ipydawg
 DAWGNode
-p94
-(dp95
-g12
-Nsg13
+p95
 (dp96
+g12
+I00
+sg13
+Nsg14
+(dp97
 S'\x04'
-p97
+p98
 (ipydawg
 DAWGNode
-p98
-(dp99
-g12
-Nsg13
+p99
 (dp100
-sS'final'
-p101
+g12
 I01
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbsg30
+sg13
+Nsg14
+(dp101
+sbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbsg31
 (ipydawg
 DAWGNode
 p102
 (dp103
 g12
-Nsg13
+I00
+sg13
+Nsg14
 (dp104
-g19
+g20
 (ipydawg
 DAWGNode
 p105
 (dp106
 g12
-Nsg13
+I00
+sg13
+Nsg14
 (dp107
-g62
+g63
 (ipydawg
 DAWGNode
 p108
 (dp109
 g12
-Nsg13
+I00
+sg13
+Nsg14
 (dp110
-g66
+g67
 (ipydawg
 DAWGNode
 p111
 (dp112
 g12
-Nsg13
+I00
+sg13
+Nsg14
 (dp113
-g70
+g71
 (ipydawg
 DAWGNode
 p114
 (dp115
 g12
-Nsg13
+I00
+sg13
+Nsg14
 (dp116
-g74
+g75
 (ipydawg
 DAWGNode
 p117
 (dp118
 g12
-Nsg13
+I00
+sg13
+Nsg14
 (dp119
-g78
+g79
 (ipydawg
 DAWGNode
 p120
 (dp121
 g12
-Nsg13
+I00
+sg13
+Nsg14
 (dp122
-g82
+g83
 (ipydawg
 DAWGNode
 p123
 (dp124
 g12
-Nsg13
+I00
+sg13
+Nsg14
 (dp125
-g86
+g87
 (ipydawg
 DAWGNode
 p126
 (dp127
 g12
-Nsg13
+I00
+sg13
+Nsg14
 (dp128
-g90
+g91
 (ipydawg
 DAWGNode
 p129
 (dp130
 g12
-Nsg13
+I00
+sg13
+Nsg14
 (dp131
-g19
+g20
 (ipydawg
 DAWGNode
 p132
 (dp133
 g12
-Nsg13
+I00
+sg13
+Nsg14
 (dp134
-g97
+g98
 (ipydawg
 DAWGNode
 p135
 (dp136
 g12
-Nsg13
-(dp137
-sg101
-I01
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
 I01
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbssg101
-I00
-sbsS'_numbers_valid'
+sg13
+Nsg14
+(dp137
+sbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbssbsS'_numbers_valid'
 p138
 I00
 sS'register'
@@ -412,22 +412,22 @@ c__builtin__
 set
 p140
 ((lp141
-g87
-ag50
-ag91
-ag63
-ag67
-ag94
-ag47
-ag98
-ag71
-ag59
-ag75
-ag53
-ag44
-ag79
-ag83
-ag56
+g99
+ag60
+ag57
+ag76
+ag84
+ag80
+ag54
+ag45
+ag88
+ag64
+ag92
+ag68
+ag48
+ag95
+ag51
+ag72
 atp142
 Rp143
 sS'wp'
@@ -511,10 +511,10 @@ DAWGNode
 p186
 (dp187
 g12
-Nsg13
-(dp188
-sg101
 I00
+sg13
+Nsg14
+(dp188
 sbsg138
 I00
 sg139
@@ -526,7 +526,7 @@ sg144
 g154
 sbsS'freq'
 p192
-S'1w'
+S'1d1u'
 p193
 sS'adress'
 p194

diff --git a/program/everything/crawler.db.backup b/program/everything/crawler.db.backup
new file mode 100644 (file)
index 0000000..74ef366
--- /dev/null
+++ b/program/everything/crawler.db.backup
@@ -0,0 +1,535 @@
+(dp0
+S'Paradiso'
+p1
+(dp2
+S'website'
+p3
+S'www.paradiso.nl'
+p4
+sS'name'
+p5
+g1
+sS'titledawg'
+p6
+(ipydawg
+DAWG
+p7
+(dp8
+S'q0'
+p9
+(ipydawg
+DAWGNode
+p10
+(dp11
+S'number'
+p12
+NsS'children'
+p13
+(dp14
+S'\x01'
+p15
+(ipydawg
+DAWGNode
+p16
+(dp17
+g12
+Nsg13
+(dp18
+S' '
+p19
+(ipydawg
+DAWGNode
+p20
+(dp21
+g12
+Nsg13
+(dp22
+S'\x02'
+p23
+(ipydawg
+DAWGNode
+p24
+(dp25
+g12
+Nsg13
+(dp26
+g19
+(ipydawg
+DAWGNode
+p27
+(dp28
+g12
+Nsg13
+(dp29
+S'-'
+p30
+(ipydawg
+DAWGNode
+p31
+(dp32
+g12
+Nsg13
+(dp33
+g19
+(ipydawg
+DAWGNode
+p34
+(dp35
+g12
+Nsg13
+(dp36
+S'\x03'
+p37
+(ipydawg
+DAWGNode
+p38
+(dp39
+g12
+Nsg13
+(dp40
+g19
+(ipydawg
+DAWGNode
+p41
+(dp42
+g12
+Nsg13
+(dp43
+g19
+(ipydawg
+DAWGNode
+p44
+(dp45
+g12
+Nsg13
+(dp46
+g19
+(ipydawg
+DAWGNode
+p47
+(dp48
+g12
+Nsg13
+(dp49
+g30
+(ipydawg
+DAWGNode
+p50
+(dp51
+g12
+Nsg13
+(dp52
+g19
+(ipydawg
+DAWGNode
+p53
+(dp54
+g12
+Nsg13
+(dp55
+g19
+(ipydawg
+DAWGNode
+p56
+(dp57
+g12
+Nsg13
+(dp58
+g19
+(ipydawg
+DAWGNode
+p59
+(dp60
+g12
+Nsg13
+(dp61
+S'L'
+p62
+(ipydawg
+DAWGNode
+p63
+(dp64
+g12
+Nsg13
+(dp65
+S'o'
+p66
+(ipydawg
+DAWGNode
+p67
+(dp68
+g12
+Nsg13
+(dp69
+S'c'
+p70
+(ipydawg
+DAWGNode
+p71
+(dp72
+g12
+Nsg13
+(dp73
+S'a'
+p74
+(ipydawg
+DAWGNode
+p75
+(dp76
+g12
+Nsg13
+(dp77
+S't'
+p78
+(ipydawg
+DAWGNode
+p79
+(dp80
+g12
+Nsg13
+(dp81
+S'i'
+p82
+(ipydawg
+DAWGNode
+p83
+(dp84
+g12
+Nsg13
+(dp85
+S'e'
+p86
+(ipydawg
+DAWGNode
+p87
+(dp88
+g12
+Nsg13
+(dp89
+S':'
+p90
+(ipydawg
+DAWGNode
+p91
+(dp92
+g12
+Nsg13
+(dp93
+g19
+(ipydawg
+DAWGNode
+p94
+(dp95
+g12
+Nsg13
+(dp96
+S'\x04'
+p97
+(ipydawg
+DAWGNode
+p98
+(dp99
+g12
+Nsg13
+(dp100
+sS'final'
+p101
+I01
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbsg30
+(ipydawg
+DAWGNode
+p102
+(dp103
+g12
+Nsg13
+(dp104
+g19
+(ipydawg
+DAWGNode
+p105
+(dp106
+g12
+Nsg13
+(dp107
+g62
+(ipydawg
+DAWGNode
+p108
+(dp109
+g12
+Nsg13
+(dp110
+g66
+(ipydawg
+DAWGNode
+p111
+(dp112
+g12
+Nsg13
+(dp113
+g70
+(ipydawg
+DAWGNode
+p114
+(dp115
+g12
+Nsg13
+(dp116
+g74
+(ipydawg
+DAWGNode
+p117
+(dp118
+g12
+Nsg13
+(dp119
+g78
+(ipydawg
+DAWGNode
+p120
+(dp121
+g12
+Nsg13
+(dp122
+g82
+(ipydawg
+DAWGNode
+p123
+(dp124
+g12
+Nsg13
+(dp125
+g86
+(ipydawg
+DAWGNode
+p126
+(dp127
+g12
+Nsg13
+(dp128
+g90
+(ipydawg
+DAWGNode
+p129
+(dp130
+g12
+Nsg13
+(dp131
+g19
+(ipydawg
+DAWGNode
+p132
+(dp133
+g12
+Nsg13
+(dp134
+g97
+(ipydawg
+DAWGNode
+p135
+(dp136
+g12
+Nsg13
+(dp137
+sg101
+I01
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I01
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbssg101
+I00
+sbsS'_numbers_valid'
+p138
+I00
+sS'register'
+p139
+c__builtin__
+set
+p140
+((lp141
+g87
+ag50
+ag91
+ag63
+ag67
+ag94
+ag47
+ag98
+ag71
+ag59
+ag75
+ag53
+ag44
+ag79
+ag83
+ag56
+atp142
+Rp143
+sS'wp'
+p144
+S'\x01 \x02 - \x03 - Locatie: \x04'
+p145
+sbsS'dloc'
+p146
+S'Grote zaal'
+p147
+sS'venue'
+p148
+S'Paradiso'
+p149
+sS'content'
+p150
+(lp151
+(lp152
+S'zaterdag 31 mei 2014  - Lentekabinet Festival Afterparty   -   Locatie: Tolhuistuin (zaal)'
+p153
+aS''
+p154
+aa(lp155
+S'<span class="uiWebviewHighlight" style="color: white; background-color: rgb(139, 0, 0);">vrijdag 4 juli 2014</span> <span class="uiWebviewHighlight" style="color: white; background-color: red;">20:30</span> - <span class="uiWebviewHighlight" style="color: white; background-color: green;">The Crimson Projekct</span>   -   Locatie: <span class="uiWebviewHighlight" style="color: white; background-color: blue;">Tolhuistuin (zaal)</span>'
+p156
+aS'Muziek rond King Crimson'
+p157
+aa(lp158
+S'<span class="uiWebviewHighlight" style="color: white; background-color: rgb(139, 0, 0);">dinsdag 10 juni 2014</span> <span class="uiWebviewHighlight" style="color: white; background-color: red;">20:30</span> - <span class="uiWebviewHighlight" style="color: white; background-color: green;">Het Ultieme Natuurkunde Feestje \xe2\x80\x93 keynote Amanda Gefter</span>'
+p159
+ag154
+aa(lp160
+S'<span class="uiWebviewHighlight" style="color: white; background-color: rgb(139, 0, 0);">dinsdag 12 augustus 2014</span> <span class="uiWebviewHighlight" style="color: white; background-color: red;">21:00</span> - <span class="uiWebviewHighlight" style="color: white; background-color: green;">Kevin Drew</span> - Locatie: <span class="uiWebviewHighlight" style="color: white; background-color: blue;">Bitterzoet</span>'
+p161
+aS'mede-oprichter Broken Social Scene solo'
+p162
+aa(lp163
+S'<span class="uiWebviewHighlight" style="color: white; background-color: rgb(139, 0, 0);">vrijdag 4 juli 2014</span> <span class="uiWebviewHighlight" style="color: white; background-color: red;">22:00</span> - <span class="uiWebviewHighlight" style="color: white; background-color: green;">Palenke Soultribe</span>'
+p164
+aS'Electronische muziek en Afro-Colombiaanse ritmes'
+p165
+aa(lp166
+S'<span class="uiWebviewHighlight" style="color: white; background-color: rgb(139, 0, 0);">maandag 3 november 2014</span> <span class="uiWebviewHighlight" style="color: white; background-color: red;">20:15</span> - <span class="uiWebviewHighlight" style="color: white; background-color: green;">Eefje de Visser: Waterwereldsteden</span> - Locatie: <span class="uiWebviewHighlight" style="color: white; background-color: blue;">Het Concertgebouw</span>'
+p167
+ag154
+aa(lp168
+S'zaterdag 27 september 2014 20:30 - A Great Big World - Locatie: Tolhuistuin (zaal)'
+p169
+aS'Hitschrijvers uit New York'
+p170
+aa(lp171
+S'zaterdag 7 juni 2014 23:00 - Benefietavond Marokkaanse Boot'
+p172
+aS'Van Amsterdam naar Tanger'
+p173
+aa(lp174
+S'donderdag 13 november 2014 19:30 - Wouter Hamel'
+p175
+aS'Sprankelende jazzy pop'
+p176
+aa(lp177
+S'vrijdag 13 juni 2014 00:00 - Legends'
+p178
+ag154
+aasS'headers'
+p179
+(lp180
+S'Title'
+p181
+aS'Summary'
+p182
+asS'summarydawg'
+p183
+(ipydawg
+DAWG
+p184
+(dp185
+g9
+(ipydawg
+DAWGNode
+p186
+(dp187
+g12
+Nsg13
+(dp188
+sg101
+I00
+sbsg138
+I00
+sg139
+g140
+((lp189
+tp190
+Rp191
+sg144
+g154
+sbsS'freq'
+p192
+S'1w'
+p193
+sS'adress'
+p194
+S'adres'
+p195
+ss.
\ No newline at end of file

diff --git a/program/everything/crawler.py b/program/everything/crawler.py
index 8a864b0..dfc70d4 100644 (file)
--- a/program/everything/crawler.py
+++ b/program/everything/crawler.py
@@ -166,6 +166,11 @@ def main():
     if len(sys.argv) == 5 and sys.argv[1] == 'test':
         cr = Crawler()
         print cr.test_entry(*sys.argv[2:])
+    elif len(sys.argv) == 5 and sys.argv[1] == 'edit':
+        cr = Crawler()
+        name, key, value = sys.argv[2:]
+        cr.entries[name][key] = value
+        cr.write()
     elif len(sys.argv) >= 2 and sys.argv[1] == 'run':
         args = sys.argv[2:]
         force = True if '-f' in args else False
@@ -194,6 +199,7 @@ def main():
     else:
         print ('{0} list'
                '{0} test crawlername title summary'
+               '{0} edit crawlername key value'
                '{0} run -f {item1 item2 ...|all}').format(sys.argv[0])
 
 if __name__ == '__main__':