CINXE.COM
HDFS Remote
<!doctype html> <html lang="en"> <head> <meta charset="utf-8"> <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"> <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"> <meta name="description" content="Remote for Hadoop Distributed Filesystem"> <meta name="author" content="Nick Craig-Wood"> <link rel="shortcut icon" type="image/png" href="/img/rclone-32x32.png"/> <script defer data-domain="rclone.org" src="https://weblog.rclone.org/js/script.js"></script> <title>HDFS Remote</title> <link rel="canonical" href="https://rclone.org/hdfs/"> <link href="/css/bootstrap.min.4.4.1.css" rel="stylesheet"> <link href="/css/font-awesome.min.5.10.2.css" rel="stylesheet"> <link href="/css/custom.css?r=-62135596800" rel="stylesheet"> </head> <body> <nav class="navbar navbar-expand-md navbar-dark bg-dark"> <a class="navbar-brand" href="/"><img class="rclone-logo" src="/img/logo_on_dark__horizontal_color.svg" alt="rclone logo"></a> <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation"> <span class="navbar-toggler-icon"></span> </button> <div class="collapse navbar-collapse" id="navbarSupportedContent"> <ul class="navbar-nav mr-auto"> <li class="nav-item active"> <a class="nav-link" href="/downloads/">Downloads</a> </li> <li class="nav-item active dropdown"> <a class="nav-link dropdown-toggle" href="#" id="navbarDropdown" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false"> Docs </a> <div class="dropdown-menu" aria-labelledby="navbarDropdown"> <a class="dropdown-item" href="/install/"><i class="fa fa-book fa-fw"></i> Installation</a> <a class="dropdown-item" href="/docs/"><i class="fa fa-book fa-fw"></i> Usage</a> <a class="dropdown-item" href="/filtering/"><i class="fa fa-book fa-fw"></i> Filtering</a> <a class="dropdown-item" href="/gui/"><i class="fa fa-book fa-fw"></i> GUI</a> <a class="dropdown-item" href="/rc/"><i class="fa fa-book fa-fw"></i> Remote Control</a> <a class="dropdown-item" href="/changelog/"><i class="fa fa-book fa-fw"></i> Changelog</a> <a class="dropdown-item" href="/bugs/"><i class="fa fa-book fa-fw"></i> Bugs</a> <a class="dropdown-item" href="/faq/"><i class="fa fa-book fa-fw"></i> FAQ</a> <a class="dropdown-item" href="/flags/"><i class="fa fa-book fa-fw"></i> Flags</a> <a class="dropdown-item" href="/licence/"><i class="fa fa-book fa-fw"></i> Licence</a> <a class="dropdown-item" href="/authors/"><i class="fa fa-book fa-fw"></i> Authors</a> <a class="dropdown-item" href="/privacy/"><i class="fa fa-book fa-fw"></i> Privacy Policy</a> </div> </li> <li class="nav-item active dropdown"> <a class="nav-link dropdown-toggle" href="#" id="navbarDropdown" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false"> Commands </a> <div class="dropdown-menu pre-scrollable" aria-labelledby="navbarDropdown"> <a class="dropdown-item" href="/commands/"><i class="fas fa-map fa-fw"></i> Overview</a> <div class="dropdown-divider"></div> <a class="dropdown-item" href="/commands/rclone/"><i class="fa fa-book fa-fw"></i> rclone</a> <a class="dropdown-item" href="/commands/rclone_about/"><i class="fa fa-book fa-fw"></i> rclone about</a> <a class="dropdown-item" href="/commands/rclone_authorize/"><i class="fa fa-book fa-fw"></i> rclone authorize</a> <a class="dropdown-item" href="/commands/rclone_backend/"><i class="fa fa-book fa-fw"></i> rclone backend</a> <a class="dropdown-item" href="/commands/rclone_bisync/"><i class="fa fa-book fa-fw"></i> rclone bisync</a> <a class="dropdown-item" href="/commands/rclone_cat/"><i class="fa fa-book fa-fw"></i> rclone cat</a> <a class="dropdown-item" href="/commands/rclone_check/"><i class="fa fa-book fa-fw"></i> rclone check</a> <a class="dropdown-item" href="/commands/rclone_checksum/"><i class="fa fa-book fa-fw"></i> rclone checksum</a> <a class="dropdown-item" href="/commands/rclone_cleanup/"><i class="fa fa-book fa-fw"></i> rclone cleanup</a> <a class="dropdown-item" href="/commands/rclone_completion/"><i class="fa fa-book fa-fw"></i> rclone completion</a> <a class="dropdown-item" href="/commands/rclone_config/"><i class="fa fa-book fa-fw"></i> rclone config</a> <a class="dropdown-item" href="/commands/rclone_copy/"><i class="fa fa-book fa-fw"></i> rclone copy</a> <a class="dropdown-item" href="/commands/rclone_copyto/"><i class="fa fa-book fa-fw"></i> rclone copyto</a> <a class="dropdown-item" href="/commands/rclone_copyurl/"><i class="fa fa-book fa-fw"></i> rclone copyurl</a> <a class="dropdown-item" href="/commands/rclone_cryptcheck/"><i class="fa fa-book fa-fw"></i> rclone cryptcheck</a> <a class="dropdown-item" href="/commands/rclone_cryptdecode/"><i class="fa fa-book fa-fw"></i> rclone cryptdecode</a> <a class="dropdown-item" href="/commands/rclone_dedupe/"><i class="fa fa-book fa-fw"></i> rclone dedupe</a> <a class="dropdown-item" href="/commands/rclone_delete/"><i class="fa fa-book fa-fw"></i> rclone delete</a> <a class="dropdown-item" href="/commands/rclone_deletefile/"><i class="fa fa-book fa-fw"></i> rclone deletefile</a> <a class="dropdown-item" href="/commands/rclone_gendocs/"><i class="fa fa-book fa-fw"></i> rclone gendocs</a> <a class="dropdown-item" href="/commands/rclone_gitannex/"><i class="fa fa-book fa-fw"></i> rclone gitannex</a> <a class="dropdown-item" href="/commands/rclone_hashsum/"><i class="fa fa-book fa-fw"></i> rclone hashsum</a> <a class="dropdown-item" href="/commands/rclone_link/"><i class="fa fa-book fa-fw"></i> rclone link</a> <a class="dropdown-item" href="/commands/rclone_listremotes/"><i class="fa fa-book fa-fw"></i> rclone listremotes</a> <a class="dropdown-item" href="/commands/rclone_ls/"><i class="fa fa-book fa-fw"></i> rclone ls</a> <a class="dropdown-item" href="/commands/rclone_lsd/"><i class="fa fa-book fa-fw"></i> rclone lsd</a> <a class="dropdown-item" href="/commands/rclone_lsf/"><i class="fa fa-book fa-fw"></i> rclone lsf</a> <a class="dropdown-item" href="/commands/rclone_lsjson/"><i class="fa fa-book fa-fw"></i> rclone lsjson</a> <a class="dropdown-item" href="/commands/rclone_lsl/"><i class="fa fa-book fa-fw"></i> rclone lsl</a> <a class="dropdown-item" href="/commands/rclone_md5sum/"><i class="fa fa-book fa-fw"></i> rclone md5sum</a> <a class="dropdown-item" href="/commands/rclone_mkdir/"><i class="fa fa-book fa-fw"></i> rclone mkdir</a> <a class="dropdown-item" href="/commands/rclone_mount/"><i class="fa fa-book fa-fw"></i> rclone mount</a> <a class="dropdown-item" href="/commands/rclone_move/"><i class="fa fa-book fa-fw"></i> rclone move</a> <a class="dropdown-item" href="/commands/rclone_moveto/"><i class="fa fa-book fa-fw"></i> rclone moveto</a> <a class="dropdown-item" href="/commands/rclone_ncdu/"><i class="fa fa-book fa-fw"></i> rclone ncdu</a> <a class="dropdown-item" href="/commands/rclone_nfsmount/"><i class="fa fa-book fa-fw"></i> rclone nfsmount</a> <a class="dropdown-item" href="/commands/rclone_obscure/"><i class="fa fa-book fa-fw"></i> rclone obscure</a> <a class="dropdown-item" href="/commands/rclone_purge/"><i class="fa fa-book fa-fw"></i> rclone purge</a> <a class="dropdown-item" href="/commands/rclone_rc/"><i class="fa fa-book fa-fw"></i> rclone rc</a> <a class="dropdown-item" href="/commands/rclone_rcat/"><i class="fa fa-book fa-fw"></i> rclone rcat</a> <a class="dropdown-item" href="/commands/rclone_rcd/"><i class="fa fa-book fa-fw"></i> rclone rcd</a> <a class="dropdown-item" href="/commands/rclone_rmdir/"><i class="fa fa-book fa-fw"></i> rclone rmdir</a> <a class="dropdown-item" href="/commands/rclone_rmdirs/"><i class="fa fa-book fa-fw"></i> rclone rmdirs</a> <a class="dropdown-item" href="/commands/rclone_selfupdate/"><i class="fa fa-book fa-fw"></i> rclone selfupdate</a> <a class="dropdown-item" href="/commands/rclone_serve/"><i class="fa fa-book fa-fw"></i> rclone serve</a> <a class="dropdown-item" href="/commands/rclone_settier/"><i class="fa fa-book fa-fw"></i> rclone settier</a> <a class="dropdown-item" href="/commands/rclone_sha1sum/"><i class="fa fa-book fa-fw"></i> rclone sha1sum</a> <a class="dropdown-item" href="/commands/rclone_size/"><i class="fa fa-book fa-fw"></i> rclone size</a> <a class="dropdown-item" href="/commands/rclone_sync/"><i class="fa fa-book fa-fw"></i> rclone sync</a> <a class="dropdown-item" href="/commands/rclone_test/"><i class="fa fa-book fa-fw"></i> rclone test</a> <a class="dropdown-item" href="/commands/rclone_touch/"><i class="fa fa-book fa-fw"></i> rclone touch</a> <a class="dropdown-item" href="/commands/rclone_tree/"><i class="fa fa-book fa-fw"></i> rclone tree</a> <a class="dropdown-item" href="/commands/rclone_version/"><i class="fa fa-book fa-fw"></i> rclone version</a> </div> </li> <li class="nav-item active dropdown"> <a class="nav-link dropdown-toggle" href="#" id="navbarDropdown" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false"> Storage Systems </a> <div class="dropdown-menu pre-scrollable" aria-labelledby="navbarDropdown"> <a class="dropdown-item" href="/overview/"><i class="fas fa-map fa-fw"></i> Overview</a> <div class="dropdown-divider"></div> <a class="dropdown-item" href="/fichier/"><i class="fa fa-archive fa-fw"></i> 1Fichier</a> <a class="dropdown-item" href="/netstorage/"><i class="fas fa-database fa-fw"></i> Akamai NetStorage</a> <a class="dropdown-item" href="/alias/"><i class="fa fa-link fa-fw"></i> Alias</a> <a class="dropdown-item" href="/s3/"><i class="fab fa-amazon fa-fw"></i> Amazon S3</a> <a class="dropdown-item" href="/b2/"><i class="fa fa-fire fa-fw"></i> Backblaze B2</a> <a class="dropdown-item" href="/box/"><i class="fa fa-archive fa-fw"></i> Box</a> <a class="dropdown-item" href="/chunker/"><i class="fa fa-cut fa-fw"></i> Chunker (splits large files)</a> <a class="dropdown-item" href="/compress/"><i class="fas fa-compress fa-fw"></i> Compress (transparent gzip compression)</a> <a class="dropdown-item" href="/combine/"><i class="fa fa-folder-plus fa-fw"></i> Combine (remotes into a directory tree)</a> <a class="dropdown-item" href="/sharefile/"><i class="fas fa-share-square fa-fw"></i> Citrix ShareFile</a> <a class="dropdown-item" href="/crypt/"><i class="fa fa-lock fa-fw"></i> Crypt (encrypts the others)</a> <a class="dropdown-item" href="/koofr/#digi-storage"><i class="fa fa-cloud fa-fw"></i> Digi Storage</a> <a class="dropdown-item" href="/dropbox/"><i class="fab fa-dropbox fa-fw"></i> Dropbox</a> <a class="dropdown-item" href="/filefabric/"><i class="fa fa-cloud fa-fw"></i> Enterprise File Fabric</a> <a class="dropdown-item" href="/filescom/"><i class="fa fa-file-alt fa-fw"></i> Files.com</a> <a class="dropdown-item" href="/ftp/"><i class="fa fa-file fa-fw"></i> FTP</a> <a class="dropdown-item" href="/gofile/"><i class="fa fa-folder fa-fw"></i> Gofile</a> <a class="dropdown-item" href="/googlecloudstorage/"><i class="fab fa-google fa-fw"></i> Google Cloud Storage</a> <a class="dropdown-item" href="/drive/"><i class="fab fa-google fa-fw"></i> Google Drive</a> <a class="dropdown-item" href="/googlephotos/"><i class="fas fa-images fa-fw"></i> Google Photos</a> <a class="dropdown-item" href="/hasher/"><i class="fa fa-check-double fa-fw"></i> Hasher (better checksums for others)</a> <a class="dropdown-item" href="/hdfs/"><i class="fa fa-globe fa-fw"></i> HDFS (Hadoop Distributed Filesystem)</a> <a class="dropdown-item" href="/hidrive/"><i class="fa fa-cloud fa-fw"></i> HiDrive</a> <a class="dropdown-item" href="/http/"><i class="fa fa-globe fa-fw"></i> HTTP</a> <a class="dropdown-item" href="/imagekit/"><i class="fa fa-cloud fa-fw"></i> ImageKit</a> <a class="dropdown-item" href="/internetarchive/"><i class="fa fa-archive fa-fw"></i> Internet Archive</a> <a class="dropdown-item" href="/jottacloud/"><i class="fa fa-cloud fa-fw"></i> Jottacloud</a> <a class="dropdown-item" href="/koofr/"><i class="fa fa-suitcase fa-fw"></i> Koofr</a> <a class="dropdown-item" href="/linkbox/"><i class="fa fa-infinity fa-fw"></i> Linkbox</a> <a class="dropdown-item" href="/mailru/"><i class="fa fa-at fa-fw"></i> Mail.ru Cloud</a> <a class="dropdown-item" href="/mega/"><i class="fa fa-archive fa-fw"></i> Mega</a> <a class="dropdown-item" href="/memory/"><i class="fas fa-memory fa-fw"></i> Memory</a> <a class="dropdown-item" href="/azureblob/"><i class="fab fa-windows fa-fw"></i> Microsoft Azure Blob Storage</a> <a class="dropdown-item" href="/azurefiles/"><i class="fab fa-windows fa-fw"></i> Microsoft Azure Files Storage</a> <a class="dropdown-item" href="/onedrive/"><i class="fab fa-windows fa-fw"></i> Microsoft OneDrive</a> <a class="dropdown-item" href="/opendrive/"><i class="fa fa-space-shuttle fa-fw"></i> OpenDrive</a> <a class="dropdown-item" href="/qingstor/"><i class="fas fa-hdd fa-fw"></i> QingStor</a> <a class="dropdown-item" href="/swift/"><i class="fa fa-space-shuttle fa-fw"></i> Openstack Swift</a> <a class="dropdown-item" href="/oracleobjectstorage/"><i class="fa fa-cloud fa-fw"></i> Oracle Object Storage</a> <a class="dropdown-item" href="/pcloud/"><i class="fa fa-cloud fa-fw"></i> pCloud</a> <a class="dropdown-item" href="/pikpak/"><i class="fa fa-cloud fa-fw"></i> PikPak</a> <a class="dropdown-item" href="/pixeldrain/"><i class="fa fa-circle fa-fw"></i> Pixeldrain</a> <a class="dropdown-item" href="/premiumizeme/"><i class="fa fa-user fa-fw"></i> premiumize.me</a> <a class="dropdown-item" href="/putio/"><i class="fas fa-parking fa-fw"></i> put.io</a> <a class="dropdown-item" href="/protondrive/"><i class="fas fa-folder fa-fw"></i> Proton Drive</a> <a class="dropdown-item" href="/quatrix/"><i class="fas fa-shield-alt fa-fw"></i> Quatrix</a> <a class="dropdown-item" href="/seafile/"><i class="fa fa-server fa-fw"></i> Seafile</a> <a class="dropdown-item" href="/sftp/"><i class="fa fa-server fa-fw"></i> SFTP</a> <a class="dropdown-item" href="/sia/"><i class="fa fa-globe fa-fw"></i> Sia</a> <a class="dropdown-item" href="/smb/"><i class="fa fa-server fa-fw"></i> SMB / CIFS</a> <a class="dropdown-item" href="/storj/"><i class="fas fa-dove fa-fw"></i> Storj</a> <a class="dropdown-item" href="/sugarsync/"><i class="fas fa-dove fa-fw"></i> SugarSync</a> <a class="dropdown-item" href="/ulozto/"><i class="fas fa-angle-double-down fa-fw"></i> Uloz.to</a> <a class="dropdown-item" href="/uptobox/"><i class="fa fa-archive fa-fw"></i> Uptobox</a> <a class="dropdown-item" href="/union/"><i class="fa fa-link fa-fw"></i> Union (merge backends)</a> <a class="dropdown-item" href="/webdav/"><i class="fa fa-server fa-fw"></i> WebDAV</a> <a class="dropdown-item" href="/yandex/"><i class="fa fa-space-shuttle fa-fw"></i> Yandex Disk</a> <a class="dropdown-item" href="/zoho/"><i class="fas fa-folder fa-fw"></i> Zoho WorkDrive</a> <div class="dropdown-divider"></div> <a class="dropdown-item" href="/local/"><i class="fas fa-hdd fa-fw"></i> The local filesystem</a> </div> </li> <li class="nav-item active"> <a class="nav-link" href="/contact/"><i class="fa fa-envelope fa-fw"></i> Contact</a> </li> <li class="nav-item active"> <a class="nav-link" href="/sponsor/"><i class="fa fa-heart heart fa-fw"></i> Sponsor</a> </li> </ul> <ul class="navbar-nav mr-auto"> <li class="nav-item active"> <a href="https://rclone.com/" target="_blank" class="nav-link bizbutton" title="Click here for support and advertising contracts"><i class="fa fa-briefcase"></i> For Business</a> </li> </ul> <form class="form-inline" name="search_form" action="https://google.com/search" target="_blank" onsubmit="on_search();"> <input name="words" value="" type="search" placeholder="search..." class="form-control form-control-sm"> <input name="q" value="" type="hidden"> <input name="as_sitesearch" value="https://rclone.org/" type="hidden"> </form> </div> </nav> <div class="container-fluid"> <div class="row"> <div class="col-md-9"> <span class="badge badge-pill badge-secondary float-right" style="margin-top: 30px; font-size: 100%" title="This feature needs Rclone v1.54 or later.">v1.54</span> <h1 id="hahahugoshortcode35s0hbhb-hdfs"><i class="fa fa-globe" aria-hidden="true"></i> HDFS</h1> <p><a href="https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html">HDFS</a> is a distributed file-system, part of the <a href="https://hadoop.apache.org/">Apache Hadoop</a> framework.</p> <p>Paths are specified as <code>remote:</code> or <code>remote:path/to/dir</code>.</p> <h2 id="configuration">Configuration</h2> <p>Here is an example of how to make a remote called <code>remote</code>. First run:</p> <pre><code> rclone config </code></pre> <p>This will guide you through an interactive setup process:</p> <pre tabindex="0"><code>No remotes found, make a new one? n) New remote s) Set configuration password q) Quit config n/s/q> n name> remote Type of storage to configure. Enter a string value. Press Enter for the default (""). Choose a number from below, or type in your own value [skip] XX / Hadoop distributed file system \ "hdfs" [skip] Storage> hdfs ** See help for hdfs backend at: https://rclone.org/hdfs/ ** hadoop name node and port Enter a string value. Press Enter for the default (""). Choose a number from below, or type in your own value 1 / Connect to host namenode at port 8020 \ "namenode:8020" namenode> namenode.hadoop:8020 hadoop user name Enter a string value. Press Enter for the default (""). Choose a number from below, or type in your own value 1 / Connect to hdfs as root \ "root" username> root Edit advanced config? (y/n) y) Yes n) No (default) y/n> n Remote config Configuration complete. Options: - type: hdfs - namenode: namenode.hadoop:8020 - username: root Keep this "remote" remote? y) Yes this is OK (default) e) Edit this remote d) Delete this remote y/e/d> y Current remotes: Name Type ==== ==== hadoop hdfs e) Edit existing remote n) New remote d) Delete remote r) Rename remote c) Copy remote s) Set configuration password q) Quit config e/n/d/r/c/s/q> q </code></pre><p>This remote is called <code>remote</code> and can now be used like this</p> <p>See all the top level directories</p> <pre><code>rclone lsd remote: </code></pre> <p>List the contents of a directory</p> <pre><code>rclone ls remote:directory </code></pre> <p>Sync the remote <code>directory</code> to <code>/home/local/directory</code>, deleting any excess files.</p> <pre><code>rclone sync --interactive remote:directory /home/local/directory </code></pre> <h3 id="setting-up-your-own-hdfs-instance-for-testing">Setting up your own HDFS instance for testing</h3> <p>You may start with a <a href="https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/SingleCluster.html">manual setup</a> or use the docker image from the tests:</p> <p>If you want to build the docker image</p> <pre tabindex="0"><code>git clone https://github.com/rclone/rclone.git cd rclone/fstest/testserver/images/test-hdfs docker build --rm -t rclone/test-hdfs . </code></pre><p>Or you can just use the latest one pushed</p> <pre tabindex="0"><code>docker run --rm --name "rclone-hdfs" -p 127.0.0.1:9866:9866 -p 127.0.0.1:8020:8020 --hostname "rclone-hdfs" rclone/test-hdfs </code></pre><p><strong>NB</strong> it need few seconds to startup.</p> <p>For this docker image the remote needs to be configured like this:</p> <pre tabindex="0"><code>[remote] type = hdfs namenode = 127.0.0.1:8020 username = root </code></pre><p>You can stop this image with <code>docker kill rclone-hdfs</code> (<strong>NB</strong> it does not use volumes, so all data uploaded will be lost.)</p> <h3 id="modification-times">Modification times</h3> <p>Time accurate to 1 second is stored.</p> <h3 id="checksum">Checksum</h3> <p>No checksums are implemented.</p> <h3 id="usage-information">Usage information</h3> <p>You can use the <code>rclone about remote:</code> command which will display filesystem size and current usage.</p> <h3 id="restricted-filename-characters">Restricted filename characters</h3> <p>In addition to the <a href="/overview/#restricted-characters">default restricted characters set</a> the following characters are also replaced:</p> <table> <thead> <tr> <th>Character</th> <th style="text-align:center">Value</th> <th style="text-align:center">Replacement</th> </tr> </thead> <tbody> <tr> <td>:</td> <td style="text-align:center">0x3A</td> <td style="text-align:center">：</td> </tr> </tbody> </table> <p>Invalid UTF-8 bytes will also be <a href="/overview/#invalid-utf8">replaced</a>.</p> <h3 id="standard-options">Standard options</h3> <p>Here are the Standard options specific to hdfs (Hadoop distributed file system).</p> <h4 id="hdfs-namenode">--hdfs-namenode</h4> <p>Hadoop name nodes and ports.</p> <p>E.g. "namenode-1:8020,namenode-2:8020,..." to connect to host namenodes at port 8020.</p> <p>Properties:</p> <ul> <li>Config: namenode</li> <li>Env Var: RCLONE_HDFS_NAMENODE</li> <li>Type: CommaSepList</li> <li>Default:</li> </ul> <h4 id="hdfs-username">--hdfs-username</h4> <p>Hadoop user name.</p> <p>Properties:</p> <ul> <li>Config: username</li> <li>Env Var: RCLONE_HDFS_USERNAME</li> <li>Type: string</li> <li>Required: false</li> <li>Examples: <ul> <li>"root" <ul> <li>Connect to hdfs as root.</li> </ul> </li> </ul> </li> </ul> <h3 id="advanced-options">Advanced options</h3> <p>Here are the Advanced options specific to hdfs (Hadoop distributed file system).</p> <h4 id="hdfs-service-principal-name">--hdfs-service-principal-name</h4> <p>Kerberos service principal name for the namenode.</p> <p>Enables KERBEROS authentication. Specifies the Service Principal Name (SERVICE/FQDN) for the namenode. E.g. "hdfs/namenode.hadoop.docker" for namenode running as service 'hdfs' with FQDN 'namenode.hadoop.docker'.</p> <p>Properties:</p> <ul> <li>Config: service_principal_name</li> <li>Env Var: RCLONE_HDFS_SERVICE_PRINCIPAL_NAME</li> <li>Type: string</li> <li>Required: false</li> </ul> <h4 id="hdfs-data-transfer-protection">--hdfs-data-transfer-protection</h4> <p>Kerberos data transfer protection: authentication|integrity|privacy.</p> <p>Specifies whether or not authentication, data signature integrity checks, and wire encryption are required when communicating with the datanodes. Possible values are 'authentication', 'integrity' and 'privacy'. Used only with KERBEROS enabled.</p> <p>Properties:</p> <ul> <li>Config: data_transfer_protection</li> <li>Env Var: RCLONE_HDFS_DATA_TRANSFER_PROTECTION</li> <li>Type: string</li> <li>Required: false</li> <li>Examples: <ul> <li>"privacy" <ul> <li>Ensure authentication, integrity and encryption enabled.</li> </ul> </li> </ul> </li> </ul> <h4 id="hdfs-encoding">--hdfs-encoding</h4> <p>The encoding for the backend.</p> <p>See the <a href="/overview/#encoding">encoding section in the overview</a> for more info.</p> <p>Properties:</p> <ul> <li>Config: encoding</li> <li>Env Var: RCLONE_HDFS_ENCODING</li> <li>Type: Encoding</li> <li>Default: Slash,Colon,Del,Ctl,InvalidUtf8,Dot</li> </ul> <h4 id="hdfs-description">--hdfs-description</h4> <p>Description of the remote.</p> <p>Properties:</p> <ul> <li>Config: description</li> <li>Env Var: RCLONE_HDFS_DESCRIPTION</li> <li>Type: string</li> <li>Required: false</li> </ul> <h2 id="limitations">Limitations</h2> <ul> <li>No server-side <code>Move</code> or <code>DirMove</code>.</li> <li>Checksums not implemented.</li> </ul> </div> <div class="col-md-3"> <div class="card"> <div class="card-header" style="padding: 5px 10px;"> Contents </div> <div class="card-body"> <nav id="TableOfContents"> <ul> <li><a href="#configuration">Configuration</a> <ul> <li><a href="#setting-up-your-own-hdfs-instance-for-testing">Setting up your own HDFS instance for testing</a></li> <li><a href="#modification-times">Modification times</a></li> <li><a href="#checksum">Checksum</a></li> <li><a href="#usage-information">Usage information</a></li> <li><a href="#restricted-filename-characters">Restricted filename characters</a></li> <li><a href="#standard-options">Standard options</a></li> <li><a href="#advanced-options">Advanced options</a></li> </ul> </li> <li><a href="#limitations">Limitations</a></li> </ul> </nav> <p></p> </div> </div> <div class="card"> <div class="card-header" style="padding: 5px 15px;"> Platinum Sponsor </div> <div class="card-body"> <a href="https://www.backblaze.com/cloud-storage?utm_source=publisher&utm_medium=paid&utm_campaign=break-free-app-storage-q224-rclone" target="_blank" rel="noopener" title="Visit rclone's sponsor Backblaze"><img src="/img/logos/backblaze.svg"></a><br /> </div> </div> <div class="card"> <div class="card-header" style="padding: 5px 15px;"> Gold Sponsor </div> <div class="card-body"> <a href="https://www.idrive.com/e2/?refer=rclone" target="_blank" rel="noopener" title="Visit rclone's sponsor IDrive e2"><img src="/img/logos/idrive_e2.svg" viewBox="0 0 60 55"></a><br /> </div> </div> <div class="card"> <div class="card-header" style="padding: 5px 10px;"> Share and Enjoy </div> <div class="card-body"> <p class="menu"> <i class="fab fa-twitter fa-fw" aria-hidden="true"></i> <a href="https://twitter.com/intent/tweet/?text=rclone%20-%20rsync%20for%20cloud%20storage%20from%20%40njcw&url=https%3A%2F%2Frclone.org" target="_blank" rel="noopener" aria-label="Share on Twitter">Twitter</a><br /> <i class="fab fa-facebook fa-fw" aria-hidden="true"></i> <a href="https://facebook.com/sharer/sharer.php?u=https%3A%2F%2Frclone.org" target="_blank" rel="noopener" aria-label="Share on Facebook">Facebook</a><br /> <i class="fab fa-reddit fa-fw" aria-hidden="true"></i> <a href="https://reddit.com/submit/?url=https%3A%2F%2Frclone.org&resubmit=true&title=rclone%20-%20rsync%20for%20cloud%20storage" target="_blank" rel="noopener" aria-label="Share on Reddit">Reddit</a><br /> <iframe src="//ghbtns.com/github-btn.html?user=rclone&repo=rclone&type=star&count=true" allowtransparency="true" frameborder="0" scrolling="no" width="120" height="20"></iframe> </p> </div> </div> <div class="card"> <div class="card-header" style="padding: 5px 15px;"> Links </div> <div class="card-body"> <p class="menu"> <i class="fa fa-comments fa-fw" aria-hidden="true"></i> <a href="https://forum.rclone.org">Rclone forum</a><br /> <i class="fab fa-github fa-fw" aria-hidden="true"></i> <a href="https://github.com/rclone/rclone">GitHub project</a><br /> <i class="fa fa-book fa-fw" aria-hidden="true"></i> <a href="https://github.com/rclone/rclone/wiki">Rclone Wiki</a><br /> <i class="fa fa-heart heart fa-fw" aria-hidden="true"></i> <a href="/sponsor/">Sponsor</a><br /> <i class="fab fa-twitter fa-fw" aria-hidden="true"></i> <a href="https://twitter.com/njcw">@njcw</a> </p> </div> </div> </div> </div> <footer> <div class="row"> <div class="col-md-10"> <div class="card card-body"> <p style="font-size: 90%;"> © <a href="https://www.craig-wood.com/nick/" target="blank">Nick Craig-Wood</a> 2014-2024<br> Source file <a href="https://github.com/rclone/rclone/blob/master/docs/content/hdfs.md" target="blank">hdfs.md</a> last updated <a href="https://github.com/rclone/rclone/commit/da25305ba0279abfdaad71ad031d36c447500f2b">2024-08-16</a><br> Uploaded with <a href="https://rclone.org">rclone</a>. Built with <a href="https://gohugo.io/" target="blank">Hugo</a>. Logo by <a href="https://twitter.com/andy23" target="blank">@andy23</a>. Served by <a href="https://caddyserver.com/" target="blank">Caddy</a>. Hosted at <a href="https://hetzner.cloud/?ref=62WcciVmlzHt" title="Get €20 in cloud credits" target="blank">Hetzner Cloud</a>. </p> </div> </div> </div> </footer> </div> <script src="/js/jquery.min.3.5.1.js"></script> <script src="/js/popper.min.1.16.0.js"></script> <script src="/js/bootstrap.min.4.4.1.js"></script> <script src="/js/custom.js?r=-62135596800"></script> <script src="/js/search.js?r=-62135596800"></script> </body> </html>