Sun Jul 3 08:26:55 UTC 2011 pix@kepibu.org

* Fix some problems discovered on fresh-load

Fri Jul 1 22:52:58 UTC 2011 pix@kepibu.org

* Expand the timing arguments run-for-a-while takes

Wed Jun 29 03:08:25 UTC 2011 pix@kepibu.org

* Wrap drakma:http-request so we can have automatic retries on timeouts

Tue Jun 28 17:03:09 UTC 2011 pix@kepibu.org

* Make #'run-for-a-while take the number of iterations as an argument

Tue Jun 28 08:05:37 UTC 2011 pix@kepibu.org

* Padding for time numbers, so we get 01 instead of 1

Tue Jun 28 08:05:09 UTC 2011 pix@kepibu.org

* Fix state-file pathname when *load-pathname* actually exists

Tue Jun 28 08:04:46 UTC 2011 pix@kepibu.org

* Consider text-decoration:none to be a spam indicator as well

Tue Jun 28 08:04:14 UTC 2011 pix@kepibu.org

* Don't consider a page updated if the last-known good is the same as the current version

Sat Jun 18 23:16:27 UTC 2011 pix@kepibu.org

* Send cliki reverter thread output to swank's standard-io (the repl in emacs)

Sat Jun 18 23:15:57 UTC 2011 pix@kepibu.org

* Add FIXME noting a behavior of cliki I wasn't expecting

--- old-claki/claki.asd 2013-07-18 20:08:14.000000000 +0000

+++ new-claki/claki.asd 2013-07-18 20:08:14.000000000 +0000

@@ -1,3 +1,3 @@

(defsystem :claki

- :depends-on (:oh-ducks :closure-html :cl-unification :drakma :alexandria :rucksack)

+ :depends-on (:oh-ducks :closure-html :cl-unification :drakma :alexandria :rucksack :local-time)

:components ((:file "claki")))

--- old-claki/claki.lisp 2013-07-18 20:08:14.000000000 +0000

+++ new-claki/claki.lisp 2013-07-18 20:08:14.000000000 +0000

@@ -6,13 +6,28 @@

#+(or) (clrhash *last-modified*)

+(defun http-request (&rest drakma-args)

+ "A wrapper around drakma:http-request which automatically retries the request

+in the event of a timeout."

+ (let ((times-failed 0))

+ (tagbody

+ :fetch-page

+ (handler-case (return-from http-request (apply #'drakma:http-request drakma-args))

+ (usocket:timeout-error ()

+ (sleep (* 60 (expt 2 times-failed)))

+ (incf times-failed)

+ (go :fetch-page))))))

+

(defun get-cliki-page (url)

"Returns a page from cliki if it has been modified since we last saw it or nil

if it has not been modified. Signals an error otherwise."

(multiple-value-bind (page status headers)

- (drakma:http-request (format nil "http://cliki.net/~a" url)

- :additional-headers (when (gethash url *last-modified*)

- `((:if-modified-since (gethash url *last-modified*)))))

+ (http-request (format nil "http://cliki.net/~a" url)

+ :additional-headers (when (gethash url *last-modified*)

+ `((:if-modified-since (gethash url *last-modified*)))))

+ ;; FIXME: this doesn't work all that well: it turns out cliki uses a single

+ ;; last-modified header for the entire wiki, rather than one per page, so we

+ ;; aren't saving ourselves (or cliki) much with this.

(cond

;; If the page hasn't been modified, no need to update

((= 304 status) nil)

@@ -54,6 +69,8 @@

("#footer > b" . #t(list ?b))) page)

(let ((current-version (oh-ducks.traversal:element-content b)))

(format t "; Page ~s modified, now at version ~a.~%" page-url current-version)

+ (when (string= (gethash page-url *last-known-good*) current-version)

+ (return-from parse-page nil))

(pushnew (list page-url current-version) *updated-pages* :test #'equal)

(dolist (link a)

(let ((url (oh-ducks.traversal:element-attribute :href link)))

@@ -73,12 +90,16 @@

(defun auto-classify (link)

"Auto-classify URLs based upon traits common to spammers."

(let ((rel (oh-ducks.traversal:element-attribute :rel link))

+ (style (oh-ducks.traversal:element-attribute :style link))

(url (oh-ducks.traversal:element-attribute :href link)))

(cond

((and (stringp rel)

(or (string-equal "follow" rel)

(string-equal "dofollow" rel)))

(setf (gethash url *spam-urls*) t))

+ ((and (stringp style)

+ (cl-ppcre:scan "text-decoration[ ]*:[ ]*none" style))

+ (setf (gethash url *spam-urls*) t))

(t nil))))

(defun request-classification (url &optional page version)

@@ -167,14 +188,14 @@

(defun revert-page (url current-version to-version)

(multiple-value-bind (page status headers)

- (drakma:http-request (format nil "http://cliki.net/edit/~a" url)

- :method :post

- :parameters `(("version" . ,current-version)

- ("T0" . "BODY")

- ("E0" . ,(get-cliki-source url to-version))

- ("summary" . "Spam detected, reverting to Known-Good.")

- ("captcha" . "lisp")

- ("name" . "Claki (Revertobot Beta)")))

+ (http-request (format nil "http://cliki.net/edit/~a" url)

+ :method :post

+ :parameters `(("version" . ,current-version)

+ ("T0" . "BODY")

+ ("E0" . ,(get-cliki-source url to-version))

+ ("summary" . "Spam detected, reverting to Known-Good.")

+ ("captcha" . "lisp")

+ ("name" . "Claki (Revertobot Beta)")))

(declare (ignore headers))

(cond

((and (= status 200)

@@ -189,7 +210,7 @@

returns the text you should POST to revert a cliki page to the given version."

(or (find-in-cache url version)

(multiple-value-bind (page status headers)

- (drakma:http-request (format nil "http://cliki.net/~a?source&v=~a" url version))

+ (http-request (format nil "http://cliki.net/~a?source&v=~a" url version))

(declare (ignore headers))

(cond

((= 200 status)

@@ -222,23 +243,27 @@

(defun seconds (s) s)

(defun minutes (m) (* (seconds 60) m))

(defun hours (h) (* (minutes 60) h))

+(defun days (d) (* (hours 24) d))

(defun plus-or-minus (x y) (+ (- x y) (random (* 2 y))))

-(defconstant +simple-time+ '(:year #\- :month #\- :day #\Space :hour #\: :min #\: :sec))

+(defconstant +simple-time+ '(:year #\- (:month 2) #\- (:day 2) #\Space (:hour 2) #\: (:min 2) #\: (:sec 2)))

(defun now () (local-time:format-timestring nil (local-time:now) :format +simple-time+))

-(defun run-for-a-while ()

+(defun run-for-a-while (how-long how-often variance)

(format t "; Beginning run at ~a~%" (now))

- (dotimes (i (* 2 24))

- (sleep (minutes (plus-or-minus 30 5)))

+ (dotimes (i (floor how-long how-often))

+ (sleep (plus-or-minus how-often variance))

(format t "; Unattended run at ~a~%" (now))

(unattended-revert-new-spam)

(save-state))

(format t "; Run ended at ~a~%~%" (now)))

-#+(or) (sb-thread:make-thread #'run-for-a-while :name "cliki reverter")

+#+(or)

+ (let ((stdout *standard-output*))

+ (sb-thread:make-thread (lambda () (let ((*standard-output* stdout)) (run-for-a-while (days 2) (minutes 30) (minutes 5))))

+ :name "cliki reverter"))



-(defvar *state-file* (or #.*load-pathname* #p"/home/pixel/repos/claki/state/"))

+(defvar *state-file* (merge-pathnames #p"state/" (directory-namestring (or #.*load-pathname* #p"/home/pixel/repos/claki/"))))

(defmacro with-rucksack-and-transaction ((rucksack) (&rest root-vars) &body body)

(with-unique-names (rest)