Sun Jul 3 08:26:55 UTC 2011 pix@kepibu.org

* Fix some problems discovered on fresh-load

- :depends-on (:oh-ducks :closure-html :cl-unification :drakma :alexandria :rucksack)

+ :depends-on (:oh-ducks :closure-html :cl-unification :drakma :alexandria :rucksack :local-time)

-(defvar *state-file* (merge-pathnames #p"state/" (directory-namestring (or #.*load-pathname* #p"/home/pixel/repos/"))))

+(defvar *state-file* (merge-pathnames #p"state/" (directory-namestring (or #.*load-pathname* #p"/home/pixel/repos/claki/"))))

Fri Jul 1 22:52:58 UTC 2011 pix@kepibu.org

* Expand the timing arguments run-for-a-while takes

+(defun days (d) (* (hours 24) d))

-(defun run-for-a-while (times)

+(defun run-for-a-while (how-long how-often variance)

- (dotimes (i times)

- (sleep (minutes (plus-or-minus 30 5)))

+ (dotimes (i (floor how-long how-often))

+ (sleep (plus-or-minus how-often variance))

- (sb-thread:make-thread (lambda () (let ((*standard-output* stdout)) (run-for-a-while (* 2 48))))

+ (sb-thread:make-thread (lambda () (let ((*standard-output* stdout)) (run-for-a-while (days 2) (minutes 30) (minutes 5))))

Wed Jun 29 03:08:25 UTC 2011 pix@kepibu.org

* Wrap drakma:http-request so we can have automatic retries on timeouts

+(defun http-request (&rest drakma-args)

+ "A wrapper around drakma:http-request which automatically retries the request

+in the event of a timeout."

+ (let ((times-failed 0))

+ (tagbody

+ :fetch-page

+ (handler-case (return-from http-request (apply #'drakma:http-request drakma-args))

+ (usocket:timeout-error ()

+ (sleep (* 60 (expt 2 times-failed)))

+ (incf times-failed)

+ (go :fetch-page))))))

+

- (drakma:http-request (format nil "http://cliki.net/~a" url)

- :additional-headers (when (gethash url *last-modified*)

- `((:if-modified-since (gethash url *last-modified*)))))

+ (http-request (format nil "http://cliki.net/~a" url)

+ :additional-headers (when (gethash url *last-modified*)

+ `((:if-modified-since (gethash url *last-modified*)))))

- (drakma:http-request (format nil "http://cliki.net/edit/~a" url)

- :method :post

- :parameters `(("version" . ,current-version)

- ("T0" . "BODY")

- ("E0" . ,(get-cliki-source url to-version))

- ("summary" . "Spam detected, reverting to Known-Good.")

- ("captcha" . "lisp")

- ("name" . "Claki (Revertobot Beta)")))

+ (http-request (format nil "http://cliki.net/edit/~a" url)

+ :method :post

+ :parameters `(("version" . ,current-version)

+ ("T0" . "BODY")

+ ("E0" . ,(get-cliki-source url to-version))

+ ("summary" . "Spam detected, reverting to Known-Good.")

+ ("captcha" . "lisp")

+ ("name" . "Claki (Revertobot Beta)")))

- (drakma:http-request (format nil "http://cliki.net/~a?source&v=~a" url version))

+ (http-request (format nil "http://cliki.net/~a?source&v=~a" url version))

Tue Jun 28 17:03:09 UTC 2011 pix@kepibu.org

* Make #'run-for-a-while take the number of iterations as an argument

-(defun run-for-a-while ()

+(defun run-for-a-while (times)

- (dotimes (i (* 2 24))

+ (dotimes (i times)

- (sb-thread:make-thread (lambda () (let ((*standard-output* stdout)) (run-for-a-while)))

+ (sb-thread:make-thread (lambda () (let ((*standard-output* stdout)) (run-for-a-while (* 2 48))))

Tue Jun 28 08:05:37 UTC 2011 pix@kepibu.org

* Padding for time numbers, so we get 01 instead of 1

-(defconstant +simple-time+ '(:year #\- :month #\- :day #\Space :hour #\: :min #\: :sec))

+(defconstant +simple-time+ '(:year #\- (:month 2) #\- (:day 2) #\Space (:hour 2) #\: (:min 2) #\: (:sec 2)))

Tue Jun 28 08:05:09 UTC 2011 pix@kepibu.org

* Fix state-file pathname when *load-pathname* actually exists

-(defvar *state-file* (or #.*load-pathname* #p"/home/pixel/repos/claki/state/"))

+(defvar *state-file* (merge-pathnames #p"state/" (directory-namestring (or #.*load-pathname* #p"/home/pixel/repos/"))))

Tue Jun 28 08:04:46 UTC 2011 pix@kepibu.org

* Consider text-decoration:none to be a spam indicator as well

+ (style (oh-ducks.traversal:element-attribute :style link))

+ ((and (stringp style)

+ (cl-ppcre:scan "text-decoration[ ]*:[ ]*none" style))

+ (setf (gethash url *spam-urls*) t))

Tue Jun 28 08:04:14 UTC 2011 pix@kepibu.org

* Don't consider a page updated if the last-known good is the same as the current version

+ (when (string= (gethash page-url *last-known-good*) current-version)

+ (return-from parse-page nil))

Sat Jun 18 23:16:27 UTC 2011 pix@kepibu.org

* Send cliki reverter thread output to swank's standard-io (the repl in emacs)

-#+(or) (sb-thread:make-thread #'run-for-a-while :name "cliki reverter")

+#+(or)

+ (let ((stdout *standard-output*))

+ (sb-thread:make-thread (lambda () (let ((*standard-output* stdout)) (run-for-a-while)))

+ :name "cliki reverter"))

Sat Jun 18 23:15:57 UTC 2011 pix@kepibu.org

* Add FIXME noting a behavior of cliki I wasn't expecting

+ ;; FIXME: this doesn't work all that well: it turns out cliki uses a single

+ ;; last-modified header for the entire wiki, rather than one per page, so we

+ ;; aren't saving ourselves (or cliki) much with this.

Sat Jun 18 23:15:27 UTC 2011 pix@kepibu.org

* Print when we start and stop a run.

+(defun now () (local-time:format-timestring nil (local-time:now) :format +simple-time+))

- (dotimes (i 20)

+ (format t "; Beginning run at ~a~%" (now))

+ (dotimes (i (* 2 24))

- (format t "; Unattented run at ~a~%" (local-time:format-timestring nil (local-time:now) :format +simple-time+))

+ (format t "; Unattended run at ~a~%" (now))

- (save-state)))

+ (save-state))

+ (format t "; Run ended at ~a~%~%" (now)))