ArchiveBot Redis layout — ArchiveBot 1.6 documentation

<!doctype html> <html class="no-js" lang="en"> <head> <base href="https://archivebot.readthedocs.io/en/latest/redis.html"> <meta charset="utf-8"> <meta name="viewport" content="width=device-width, initial-scale=1.0"> <title>ArchiveBot Redis layout — ArchiveBot 1.6 documentation</title> <script type="text/javascript" src="_static/js/modernizr.min.js"></script> <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script> <script type="text/javascript" src="_static/jquery.js"></script> <script type="text/javascript" src="_static/underscore.js"></script> <script type="text/javascript" src="_static/doctools.js"></script> <script type="text/javascript" src="_static/language_data.js"></script> <script type="text/javascript" src="_static/js/theme.js"></script> <link rel="stylesheet" href="_static/css/theme.css" type="text/css"> <link rel="stylesheet" href="_static/pygments.css" type="text/css"> <link rel="index" title="Index" href="genindex.html"> <link rel="search" title="Search" href="search.html"> <link rel="next" title="ArchiveBot Administration" href="admin.html"> <link rel="prev" title="Commands" href="commands.html"> <script type="application/json" id="READTHEDOCS_DATA">{"ad_free": false, "api_host": "https://readthedocs.org", "builder": "sphinx", "canonical_url": null, "docroot": "/doc/", "features": {"docsearch_disabled": false}, "global_analytics_code": "UA-17997319-1", "language": "en", "page": "redis", "programming_language": "py", "project": "archivebot", "proxied_api_host": "/_", "source_suffix": ".rst", "subprojects": {}, "theme": "sphinx_rtd_theme", "user_analytics_code": "", "version": "latest"}</script> <script type="text/javascript"> READTHEDOCS_DATA = JSON.parse(document.getElementById('READTHEDOCS_DATA').innerHTML); </script> <script async type="text/javascript" src="/_/static/javascript/readthedocs-addons.js"></script> <meta name="readthedocs-project-slug" content="archivebot"> <meta name="readthedocs-version-slug" content="latest"> <meta name="readthedocs-resolver-filename" content="/redis.html"> <meta name="readthedocs-http-status" content="200"> <meta http-equiv="X-Translated-By" content="Google"> <meta http-equiv="X-Translated-To" content="en"> <script type="text/javascript" src="https://www.gstatic.com/_/translate_http/_/js/k=translate_http.tr.en_GB.WgGHrg8C9fE.O/am=DgY/d=1/rs=AN8SPfpNfjzpGCAsUUJ5X-GCaxSfec_Eng/m=corsproxy" data-sourceurl="https://archivebot.readthedocs.io/en/latest/redis.html"></script> <link href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20..48,100..700,0..1,-50..200" rel="stylesheet"> <script type="text/javascript" src="https://www.gstatic.com/_/translate_http/_/js/k=translate_http.tr.en_GB.WgGHrg8C9fE.O/am=DgY/d=1/exm=corsproxy/ed=1/rs=AN8SPfpNfjzpGCAsUUJ5X-GCaxSfec_Eng/m=phishing_protection" data-phishing-protection-enabled="false" data-forms-warning-enabled="true" data-source-url="https://archivebot.readthedocs.io/en/latest/redis.html"></script> <meta name="robots" content="none"> </head> <body class="wy-body-for-nav"> <script type="text/javascript" src="https://www.gstatic.com/_/translate_http/_/js/k=translate_http.tr.en_GB.WgGHrg8C9fE.O/am=DgY/d=1/exm=corsproxy,phishing_protection/ed=1/rs=AN8SPfpNfjzpGCAsUUJ5X-GCaxSfec_Eng/m=navigationui" data-environment="prod" data-proxy-url="https://archivebot-readthedocs-io.translate.goog" data-proxy-full-url="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB" data-source-url="https://archivebot.readthedocs.io/en/latest/redis.html" data-source-language="auto" data-target-language="en" data-display-language="en-GB" data-detected-source-language="en" data-is-source-untranslated="false" data-source-untranslated-url="https://translate.google.com/website?sl=auto&tl=en&hl=en-GB&u=https://archivebot.readthedocs.io/en/latest/redis.html&anno=2" data-client="tr"></script> <div class="wy-grid-for-nav"> <nav data-toggle="wy-nav-shift" class="wy-nav-side"> <div class="wy-side-scroll"> <div class="wy-side-nav-search"><a href="https://archivebot-readthedocs-io.translate.goog/en/latest/index.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB" class="icon icon-home"> ArchiveBot </a> <div class="version"> latest </div> <div role="search"> <form id="rtd-search-form" class="wy-form" action="search.html" method="get"><input type="text" name="q" placeholder="Search docs"> <input type="hidden" name="check_keywords" value="yes"> <input type="hidden" name="area" value="default"> </form> </div> </div> <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="https://archivebot-readthedocs-io.translate.goog/en/latest/commands.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB">Commands</a></li> <li class="toctree-l1 current"><a class="current reference internal" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#">ArchiveBot Redis layout</a> <ul> <li class="toctree-l2"><a class="reference internal" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#connection">Connection</a></li> <li class="toctree-l2"><a class="reference internal" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#pipeline-pipeline-id"><code class="docutils literal notranslate"><span class="pre">pipeline:PIPELINE_ID</span></code></a></li> <li class="toctree-l2"><a class="reference internal" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#hash-keys">Hash keys</a></li> <li class="toctree-l2"><a class="reference internal" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#ident-i-e-a-z0-9-25"><code class="docutils literal notranslate"><span class="pre">IDENT</span></code> (i.e. <code class="docutils literal notranslate"><span class="pre">[a-z0-9]{25,}</span></code>)</a></li> <li class="toctree-l2"><a class="reference internal" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#hash-keys-1">Hash keys</a></li> <li class="toctree-l2"><a class="reference internal" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#ident-ignores"><code class="docutils literal notranslate"><span class="pre">IDENT_ignores</span></code></a></li> <li class="toctree-l2"><a class="reference internal" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#ident-log"><code class="docutils literal notranslate"><span class="pre">IDENT_log</span></code></a></li> <li class="toctree-l2"><a class="reference internal" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#pipelines"><code class="docutils literal notranslate"><span class="pre">pipelines</span></code></a></li> <li class="toctree-l2"><a class="reference internal" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#jobs-completed-jobs-aborted-jobs-failed"><code class="docutils literal notranslate"><span class="pre">jobs_completed</span></code>, <code class="docutils literal notranslate"><span class="pre">jobs_aborted</span></code>, <code class="docutils literal notranslate"><span class="pre">jobs_failed</span></code></a></li> <li class="toctree-l2"><a class="reference internal" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#tweets-done-tweets-queue"><code class="docutils literal notranslate"><span class="pre">tweets:done</span></code>, <code class="docutils literal notranslate"><span class="pre">tweets:queue</span></code></a></li> <li class="toctree-l2"><a class="reference internal" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#pubsub-channels">Pubsub channels</a></li> <li class="toctree-l2"><a class="reference internal" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#updates"><code class="docutils literal notranslate"><span class="pre">updates</span></code></a></li> <li class="toctree-l2"><a class="reference internal" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#archivebot-job-ident"><code class="docutils literal notranslate"><span class="pre">archivebot:job:IDENT</span></code></a></li> </ul></li> <li class="toctree-l1"><a class="reference internal" href="https://archivebot-readthedocs-io.translate.goog/en/latest/admin.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB">ArchiveBot Administration</a></li> </ul> </div> </div> </nav> <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> <nav class="wy-nav-top" aria-label="top navigation"><i data-toggle="wy-nav-top" class="fa fa-bars"></i> <a href="https://archivebot-readthedocs-io.translate.goog/en/latest/index.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB">ArchiveBot</a> </nav> <div class="wy-nav-content"> <div class="rst-content"> <div role="navigation" aria-label="breadcrumbs navigation"> <ul class="wy-breadcrumbs"> <li><a href="https://archivebot-readthedocs-io.translate.goog/en/latest/index.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB">Docs</a> »</li> <li>ArchiveBot Redis layout</li> <li class="wy-breadcrumbs-aside"><a href="https://translate.google.com/website?sl=auto&tl=en&hl=en-GB&u=https://github.com/ArchiveTeam/ArchiveBot/blob/master/doc/redis.rst" class="fa fa-github"> Edit on GitHub</a></li> </ul> <hr> </div> <div role="main" class="document" itemscope itemtype="http://schema.org/Article"> <div itemprop="articleBody"> <div class="section" id="archivebot-redis-layout"> <h1>ArchiveBot Redis layout<a class="headerlink" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#archivebot-redis-layout" title="Permalink to this headline">¶</a></h1> <p>The ArchiveBot pipelines and backend share a single Redis database. This document describes the keys in that database.</p> <p>Keys do not follow any namespace-prefixing convention; ArchiveBot assumes it has full control over the database.</p> <div class="section" id="connection"> <h2>Connection<a class="headerlink" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#connection" title="Permalink to this headline">¶</a></h2> <p>Pipelines connect directly to the Redis database, typically over SSH or spiped. The backend connects the same way. There is no access control from either side.</p> </div> <div class="section" id="pipeline-pipeline-id"> <h2><code class="docutils literal notranslate"><span class="pre">pipeline:PIPELINE_ID</span></code><a class="headerlink" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#pipeline-pipeline-id" title="Permalink to this headline">¶</a></h2> <p>Type: hash</p> <p>Keys matching this form describe pipelines. PIPELINE_ID is a hexadecimal number that is generated by a pipeline process on startup. The pipeline process periodically updates its data while it runs.</p> </div> <div class="section" id="hash-keys"> <h2>Hash keys<a class="headerlink" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#hash-keys" title="Permalink to this headline">¶</a></h2> <table border="1" class="docutils"> <colgroup> <col width="22%"> <col width="19%"> <col width="59%"> </colgroup> <thead valign="bottom"> <tr class="row-odd"> <th class="head">Key</th> <th class="head">Intended type</th> <th class="head">Meaning</th> </tr> </thead> <tbody valign="top"> <tr class="row-even"> <td>disk_usage</td> <td>Decimal</td> <td>% of the pipeline’s filesystem in use</td> </tr> <tr class="row-odd"> <td>disk_available</td> <td>Integer</td> <td>Bytes available on the pipeline’s filesystem</td> </tr> <tr class="row-even"> <td>fqdn</td> <td>String</td> <td>FQDN of the host running the pipeline</td> </tr> <tr class="row-odd"> <td>hostname</td> <td>String</td> <td>Short name of the host</td> </tr> <tr class="row-even"> <td>id</td> <td>String</td> <td>The pipeline’s ID; always matches the hash key</td> </tr> <tr class="row-odd"> <td>load_average_1m</td> <td>Decimal</td> <td>Load average over the past minute</td> </tr> <tr class="row-even"> <td>load_average_5m</td> <td>Decimal</td> <td>“” “” “” “” over the past 5 minutes</td> </tr> <tr class="row-odd"> <td>load_average_15m</td> <td>Decimal</td> <td>“” “” “” “” over the past 15 minutes</td> </tr> <tr class="row-even"> <td>mem_available</td> <td>Integer</td> <td>Bytes of memory available on the host</td> </tr> <tr class="row-odd"> <td>mem_usage</td> <td>Decimal</td> <td>% memory in use on the host</td> </tr> <tr class="row-even"> <td>nickname</td> <td>String</td> <td>The pipeline nickname</td> </tr> <tr class="row-odd"> <td>pid</td> <td>Integer</td> <td>The PID of the pipeline process</td> </tr> <tr class="row-even"> <td>python</td> <td>String</td> <td>The version of Python running the pipeline</td> </tr> <tr class="row-odd"> <td>ts</td> <td>UNIX timestamp</td> <td>The last time this pipeline record was updated</td> </tr> <tr class="row-even"> <td>version</td> <td>String</td> <td>The pipeline’s version</td> </tr> </tbody> </table> </div> <div class="section" id="ident-i-e-a-z0-9-25"> <h2><code class="docutils literal notranslate"><span class="pre">IDENT</span></code> (i.e. <code class="docutils literal notranslate"><span class="pre">[a-z0-9]{25,}</span></code>)<a class="headerlink" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#ident-i-e-a-z0-9-25" title="Permalink to this headline">¶</a></h2> <p>Type: hash</p> <p>These are job records. These are the most common record type in ArchiveBot’s database.</p> <p>Parts of this record are frequently modified by both the backend and pipeline:</p> <ul class="simple"> <li>whenever a response is recorded</li> <li>whenever job settings are changed</li> </ul> </div> <div class="section" id="hash-keys-1"> <h2>Hash keys<a class="headerlink" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#hash-keys-1" title="Permalink to this headline">¶</a></h2> <table border="1" class="docutils"> <colgroup> <col width="23%"> <col width="13%"> <col width="64%"> </colgroup> <thead valign="bottom"> <tr class="row-odd"> <th class="head">Key</th> <th class="head">Intended type</th> <th class="head">Meaning</th> </tr> </thead> <tbody valign="top"> <tr class="row-even"> <td>bytes_downloaded</td> <td>Integer</td> <td>Bytes downloaded from the target site</td> </tr> <tr class="row-odd"> <td>concurrency</td> <td>Integer</td> <td>Current number of concurrent downloaders</td> </tr> <tr class="row-even"> <td>death_timer</td> <td>Integer</td> <td>Number of liveness checks that have gone without a response</td> </tr> <tr class="row-odd"> <td>delay_max</td> <td>Integer</td> <td>Maximum delay between two requests on a downloader in ms</td> </tr> <tr class="row-even"> <td>delay_min</td> <td>Integer</td> <td>Minimum delay between two requests on a downloader in ms</td> </tr> <tr class="row-odd"> <td>error_count</td> <td>Integer</td> <td>Number of error (i.e. 4xx, 5xx) responses encountered</td> </tr> <tr class="row-even"> <td>fetch_depth</td> <td>String</td> <td>“shallow” for !ao jobs; “inf” for !a jobs</td> </tr> <tr class="row-odd"> <td>finished_at</td> <td>UNIX ts w/ frac</td> <td>When the job finished; not present if the job is running</td> </tr> <tr class="row-even"> <td>heartbeat</td> <td>Integer</td> <td>Set by the pipeline; incremented once per heartbeat</td> </tr> <tr class="row-odd"> <td>ignore_patterns_set_key</td> <td>String</td> <td>The key storing this job’s ignore patterns</td> </tr> <tr class="row-even"> <td>items_downloaded</td> <td>Integer</td> <td>Number of 2xx/3xx responses</td> </tr> <tr class="row-odd"> <td>items_queued</td> <td>Integer</td> <td>Number of URLs encountered in the job</td> </tr> <tr class="row-even"> <td>last_acknowledged_heartbeat</td> <td>Integer</td> <td>Set by the backend; is the last heartbeat received</td> </tr> <tr class="row-odd"> <td>last_analyzed_log_entry</td> <td>Integer</td> <td>The last log entry index analyzed by the backend [1]</td> </tr> <tr class="row-even"> <td>last_broadcasted_log_entry</td> <td>Integer</td> <td>“” “” “” “” “” “” “” “” broadcasted over the firehose [1]</td> </tr> <tr class="row-odd"> <td>last_trimmed_log_entry</td> <td>Integer</td> <td>“” “” “” “” “” “” “” “” trimmed by the log trimmer [1]</td> </tr> <tr class="row-even"> <td>log_key</td> <td>String</td> <td>The key storing this job’s log messages</td> </tr> <tr class="row-odd"> <td>log_score</td> <td>Integer</td> <td>The current log entry index</td> </tr> <tr class="row-even"> <td>next_watermark</td> <td>Integer</td> <td>A threshold for number of queued URLs; currently unused</td> </tr> <tr class="row-odd"> <td>pipeline_id</td> <td>String</td> <td>The pipeline running this job; corresponds to a pipeline:* key</td> </tr> <tr class="row-even"> <td>queued_at</td> <td>UNIX ts w/ frac</td> <td>When this job was queued</td> </tr> <tr class="row-odd"> <td>r1xx</td> <td>Integer</td> <td>Number of 1xx responses</td> </tr> <tr class="row-even"> <td>r2xx</td> <td>Integer</td> <td>“” “” “” 2xx responses</td> </tr> <tr class="row-odd"> <td>r3xx</td> <td>Integer</td> <td>“” “” “” 3xx responses</td> </tr> <tr class="row-even"> <td>r4xx</td> <td>Integer</td> <td>“” “” “” 4xx responses</td> </tr> <tr class="row-odd"> <td>r5xx</td> <td>Integer</td> <td>“” “” “” 5xx responses</td> </tr> <tr class="row-even"> <td>runk</td> <td>Integer</td> <td>“” “” “” responses with unknown HTTP status code</td> </tr> <tr class="row-odd"> <td>recorded_at</td> <td>UNIX ts w/ frac</td> <td><em>Deprecated.</em> When this job was logged to ArchiveBot’s CouchDB</td> </tr> <tr class="row-even"> <td>settings_age</td> <td>Integer</td> <td>Job settings version; incremented for each settings change</td> </tr> <tr class="row-odd"> <td>slug</td> <td>String</td> <td>WARC/JSON base filename [2]</td> </tr> <tr class="row-even"> <td>started_at</td> <td>UNIX ts w/ frac</td> <td>When this job was started by a pipeline</td> </tr> <tr class="row-odd"> <td>started_by</td> <td>String</td> <td>The user (typically an IRC nick) that submitted the job</td> </tr> <tr class="row-even"> <td>started_in</td> <td>String</td> <td>Where the job was started (typically an IRC channel)</td> </tr> <tr class="row-odd"> <td>suppress_ignore_reports</td> <td>Boolean</td> <td>Whether ignore pattern matches should be reported</td> </tr> <tr class="row-even"> <td>ts</td> <td>UNIX ts w/ frac</td> <td>Last update received from a pipeline for this job</td> </tr> <tr class="row-odd"> <td>url</td> <td>String</td> <td>The URL for this job: either the target or a URL file (for !ao < and !a <)</td> </tr> <tr class="row-even"> <td>user_agent</td> <td>String</td> <td>The user-agent to spoof; null if we should use the default agent</td> </tr> </tbody> </table> <p>[1]: The expected relationship between these values is</p> <p>last_analyzed_log_entry <= last_broadcasted_log_entry <= last_trimmed_log_entry</p> <p>[2]: Usually looks like “twitter.com-inf”. The date, time, WARC sequence, extension, etc. are all appended by the pipeline.</p> </div> <div class="section" id="ident-ignores"> <h2><code class="docutils literal notranslate"><span class="pre">IDENT_ignores</span></code><a class="headerlink" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#ident-ignores" title="Permalink to this headline">¶</a></h2> <p>Type: set</p> <p>Ignore patterns for the identified job. Each ignore pattern is a Python regex.</p> </div> <div class="section" id="ident-log"> <h2><code class="docutils literal notranslate"><span class="pre">IDENT_log</span></code><a class="headerlink" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#ident-log" title="Permalink to this headline">¶</a></h2> <p>Type: zset</p> <p>Log entries generated for a job by the wpull hooks or pipeline stdout capture are sent here. The backend is notified of new entries in this set when the pipeline publishes the job ident on the <code class="docutils literal notranslate"><span class="pre">updates</span></code> channel.</p> </div> <div class="section" id="pipelines"> <h2><code class="docutils literal notranslate"><span class="pre">pipelines</span></code><a class="headerlink" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#pipelines" title="Permalink to this headline">¶</a></h2> <p>Type: list</p> <p><em>Deprecated.</em> This list contains pipeline names, and is still modified by pipelines, but no pipeline listing uses it.</p> </div> <div class="section" id="jobs-completed-jobs-aborted-jobs-failed"> <h2><code class="docutils literal notranslate"><span class="pre">jobs_completed</span></code>, <code class="docutils literal notranslate"><span class="pre">jobs_aborted</span></code>, <code class="docutils literal notranslate"><span class="pre">jobs_failed</span></code><a class="headerlink" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#jobs-completed-jobs-aborted-jobs-failed" title="Permalink to this headline">¶</a></h2> <p>Type: string</p> <p>These keys store counts of completed, aborted, and failed jobs, respectively.</p> <p>A completed job is a job that made it through the entire ArchiveBot pipeline. An aborted job is a job that was terminated using <code class="docutils literal notranslate"><span class="pre">!abort</span></code>. A failed job is a job that crashed and was reaped using the internal console.</p> </div> <div class="section" id="tweets-done-tweets-queue"> <h2><code class="docutils literal notranslate"><span class="pre">tweets:done</span></code>, <code class="docutils literal notranslate"><span class="pre">tweets:queue</span></code><a class="headerlink" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#tweets-done-tweets-queue" title="Permalink to this headline">¶</a></h2> <p>Type: zset</p> <p>These are used by ArchiveBot’s Twitter tweeter. They store tweets that were tweeted and tweets in the to-post queue, respectively.</p> </div> <div class="section" id="pubsub-channels"> <h2>Pubsub channels<a class="headerlink" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#pubsub-channels" title="Permalink to this headline">¶</a></h2> </div> <div class="section" id="updates"> <h2><code class="docutils literal notranslate"><span class="pre">updates</span></code><a class="headerlink" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#updates" title="Permalink to this headline">¶</a></h2> <p>Whenever a pipeline has new log entries for a job, it publishes that job’s ident to this channel.</p> </div> <div class="section" id="archivebot-job-ident"> <h2><code class="docutils literal notranslate"><span class="pre">archivebot:job:IDENT</span></code><a class="headerlink" href="https://archivebot-readthedocs-io.translate.goog/en/latest/redis.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB#archivebot-job-ident" title="Permalink to this headline">¶</a></h2> <p>There exists one of these channels per job.</p> <p>When settings are updated for that job, the new settings age is published via this channel. The job’s settings listener receives the new version. If the new version is greater than the current version, the new settings are read from Redis and applied.</p> </div> </div> </div> </div> <footer> <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"><a href="https://archivebot-readthedocs-io.translate.goog/en/latest/admin.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB" class="btn btn-neutral float-right" title="ArchiveBot Administration" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a> <a href="https://archivebot-readthedocs-io.translate.goog/en/latest/commands.html?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en-GB" class="btn btn-neutral float-left" title="Commands" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a> </div> <hr> <div role="contentinfo"> <p>© Copyright 2014, ArchiveTeam <span class="commit"> Revision <code>b22627e4</code>. </span></p> </div> Built with <a href="https://translate.google.com/website?sl=auto&tl=en&hl=en-GB&u=http://sphinx-doc.org/">Sphinx</a> using a <a href="https://translate.google.com/website?sl=auto&tl=en&hl=en-GB&u=https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://translate.google.com/website?sl=auto&tl=en&hl=en-GB&u=https://readthedocs.org">Read the Docs</a>. </footer> </div> </div> </section> </div> <script type="text/javascript"> jQuery(function () { SphinxRtdTheme.Navigation.enable(true); }); </script> <script>function gtElInit() {var lib = new google.translate.TranslateService();lib.translatePage('en', 'en', function () {});}</script> <script src="https://translate.google.com/translate_a/element.js?cb=gtElInit&hl=en-GB&client=wt" type="text/javascript"></script> </body> </html>

CINXE.COM

ArchiveBot Redis layout — ArchiveBot 1.6 documentation