mirror of
https://github.com/ohmyzsh/ohmyzsh.git
synced 2026-03-20 03:13:33 +01:00
65 lines
2.5 KiB
Bash
Executable file
65 lines
2.5 KiB
Bash
Executable file
#!/bin/zsh
|
|
#
|
|
# scrape-norris.zsh
|
|
#
|
|
# Scrape Chuck Norris facts from chucknorrisfacts.com to fortune file format.
|
|
#
|
|
# Usage:
|
|
# ./tools/scrape-norris.zsh [n_pages_to_scrape]
|
|
#
|
|
# You may need to do `zsh ./tools/scrape-norrish.zsh ...` if the file isn't executable
|
|
# or your zsh is at a nonstandard location.
|
|
#
|
|
# This rebuilds the chucknorris source file from the source on the web.
|
|
#
|
|
# The output file format is the '\n%'-separated string file format used by fortune's
|
|
# `strfile` utility.
|
|
#
|
|
# This is a development tool. You only need to run it if you're a maintainer of the
|
|
# chucknorris plugin, and want to modify the main fortune file for checking back in
|
|
# to the distribution. If you're a normal user, the source file is already there and
|
|
# you don't need to use this.
|
|
#
|
|
# This script was created by looking at the URLs used by the target website.
|
|
# There's no public API. This is a total hack and is liable to break at any time.
|
|
# All the parsing code is specific to the format used by the chucknorris.com page
|
|
# as of 2/26/2015.
|
|
#
|
|
# This script is intended to be executable and run in its own shell. It assumes it's
|
|
# independent, and leaks variables whenever it wants.
|
|
#
|
|
# Dependencies: perl, fortune
|
|
|
|
if [[ -n $1 ]]; then
|
|
npages=$1
|
|
else
|
|
npages=20
|
|
fi
|
|
outfile="fortunes/chucknorris"
|
|
# Page that presents
|
|
chuckpage='http://www.chucknorrisfacts.com/all-chuck-norris-facts'
|
|
rm $outfile
|
|
echo Fetching from Chuck repository: $chuckpage
|
|
echo Fetching $npages pages worth of fortunes
|
|
for (( i = 1; i <= $npages; i++ )) do
|
|
curl -s $chuckpage\?page\=$i | perl -MHTML::Entities -ne \
|
|
'/<span class="field-content"><a href.*?>(.*?)</m && print decode_entities("$1\n%\n")' \
|
|
>> $outfile
|
|
done
|
|
nlines=$(wc -l $outfile | perl -ne '/(\d+)/ && print "$1"')
|
|
(( nfacts = nlines / 2 ))
|
|
echo Fetched $nfacts Chuck Norris facts
|
|
|
|
echo New fortune source file built at $outfile with $nfacts Chuck Norris facts
|
|
|
|
# Validate outfile format using fortune
|
|
dummyfile=$(mktemp -t chucknorris-scrape)
|
|
nparsedlines=$(strfile $outfile $dummyfile | perl -ne '/There were (\d+) strings/m && print $1')
|
|
if [[ $nparsedlines -ne $nfacts ]]; then
|
|
echo "WARNING: DATA INTEGRITY CHECK FAILURE"
|
|
echo "WARNING: Got $nfacts sayings but fortune saw $nparsedlines lines."
|
|
echo "WARNING: This probably means I had a parsing error and the new file is bad."
|
|
echo "WARNING: Do not commit these changes without manual review and verification."
|
|
echo "WARNING: Failure to review properly may endanger correctness of the Norris database!"
|
|
fi
|
|
|