Filter Cyrillic texts with Ukrainian letters from CC WET files

stage1.lisp

(let (cur)
  (with-output-to-string (out)
    (loop :for line := (read-line *standard-input* nil) :while line :do
      (cond
        ((string= #.(format nil "WARC/1.0~C" #\Return) line)
         (setf cur :header)
         (let* ((outstr (string-trim '(#\Newline #\Return #\Linefeed)
                                     (get-output-stream-string out)))
                (count-i (count-if (lambda (char)
                                     (member char '(#\і #\ї #\є #\І #\Ї #\Є #\Ґ #\ґ)))
                                   outstr)))
           (when (and (plusp count-i)
                      (> (+ count-i
                            (count-if (lambda (char)
                                        (or (char<= #\а char #\я)
                                            (char<= #\А char #\Я)))
                                      outstr))
                         (floor (length outstr) 2)))
             (write-line outstr)
             (write-line "===(=^.^=)==="))))
        ((and (eql :header cur)
              (string= #.(format nil "~C" #\Return) line))
         (setf cur :body))
        ((eql :body cur)
         (write-line line out))))))

To run:

$ cat ../input/test-100.wet | parallel -j 24 'curl -L https://commoncrawl.s3.amazonaws.com/{} | gzip -dc | sbcl --script stage1.lisp > `echo {} | sed "s/\//_/g"`.txt'