cloudcat

Cloudcat

Cloudcat is a tool for extracting structured data from websites using YAML configuration and the syntax rule is extensible.

CLI example

run analyze a model

cat << EOF | cloudcat run -m -
source:
  name: HackerNews
  http: https://news.ycombinator.com/best
  timeout: 60s
schema:
  type: array
  init:
    - gq: "#hnmain tbody -> slice(2) -> child('tr:not(.spacer,.morespace,:last-child)')"
      js: |
        content?.reduce((acc, v, i, arr) => {
          if (i % 2 === 0) {
            acc.push(arr.slice(i, i + 2).join(''));
          }
          return acc;
        }, []);
  properties:
    index:
      type: integer
      rule:
        - gq: .rank
          regex: /[^\d]/
    title: { gq: .titleline>:first-child }
    by: { gq: .hnuser }
    age: { gq: .age }
    comments:
      type: integer
      rule:
        - gq: .subline>:last-child
          regex: /[^\d]/
EOF

run a js script

cat << EOF | cloudcat run -s -
const http = require('cloudcat/http');
let res = http.get('https://news.ycombinator.com/best');
let stories = cat.getElements('gq', res.string(), "#hnmain tbody -> slice(2) -> child('tr:not(.spacer,.morespace,:last-child)')");
stories?.reduce((acc, v, i, arr) => {
    if (i % 2 === 0) {
        let item = arr.slice(i, i + 2).join('');
        let index = cat.getString('gq', item, '.rank');
        let title = cat.getString('gq', item, '.titleline>:first-child');
        let by = cat.getString('gq', item, '.hnuser');
        let age = cat.getString('gq', item, '.age');
        let comments = cat.getString('gq', item, '.subline>:last-child');
        acc.push({
            index: parseInt(index?.replace(/[^\d]+/g, ''), 10),
            title: title,
            by: by,
            age: age,
            comments: parseInt(comments?.replace(/[^\d]+/g, ''), 10)
        });
    }
    return acc;
}, []);
EOF

Documentation

See Wiki

License

cloudcat is distributed under the AGPL-3.0 license.