outputHandlerLibssh.cpp

Go to the documentation of this file.

#include "Options.h"
#include "outputHandlerLibssh.h"
#include "block.h"
#include "copyRequestTypes.h"
#include "ewmscp.h"
#include "throwcall.h"
#include "errMsgQueue.h"
#include "timer.h"
#include <libgen.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <vector>
#include <fcntl.h>
 
 
namespace outputHandler {
    decltype(libssh::factory) libssh::factory("sftp");
 
    static libsshCommon::sshOptions outputConfig("sshOut");
 
    libssh::libssh():
        libsshCommon(outputConfig) {
    }
 
 
 
    std::unique_ptr<base::writer> libssh::newWriter(const std::string& path,
            bool mightAppend,
            size_t sourceSize,
            size_t readBlockSize,
            copyRequest::stateType &state,
            bool noWrite,
            std::unique_ptr<ioHandle::attrDataType> attrData,
            std::unique_ptr<acl::list> aclData) {
        return std::unique_ptr<base::writer>(new writerLibssh(path,
                                             *this,
                                             mightAppend,
                                             sourceSize, readBlockSize,
                                             state, noWrite, std::move(attrData),
                                             std::move(aclData)));
    }
 
    void libssh::ensureParentDirs(const std::string& path,
                                  const std::string& srcPath,
                                  inputHandler::base* InputHandler) {
        std::vector<std::remove_reference<decltype(path)>::type::value_type> disposable_buffer(path.c_str(), path.c_str() + path.size() + 1);
        std::string dir(dirname(disposable_buffer.data()));
        static timer::anchor a("sftp_stat");
        timer::instanceUnscoped i(a);
        auto stat = getUniquePtr(sftp_stat(sftp, dir.c_str()), sftp_attributes_free);
        i.stop();
        
        if (stat == nullptr) {
            if (sftp_get_error(sftp) == SSH_FX_NO_SUCH_FILE) {
                ensureParentDirs(dir, srcPath, InputHandler);
                {
                    timerInst(sftp_mkdir);
                    auto result = sftp_mkdir(sftp, dir.c_str(), 0777u);
 
                    if (result != 0 && sftp_get_error(sftp) == SSH_FX_FILE_ALREADY_EXISTS) {
                        return; // we lost a race with another copy process...
                    }
 
                    throwcall::sftp::good0(result, *this, "can't create directory ", dir);
                }
                if (gid != -1 || uid != -1) {
                    timerInst(sftp_chown);
                    throwcall::sftp::good0(sftp_chown(sftp, dir.c_str(), uid, gid), *this,
                                           "can't set owner/group (", uid, ",", gid, ") on ", dir);
                }
            }
        } else if (stat->type != SSH_FILEXFER_TYPE_DIRECTORY) {
            errMsg::emit(errMsg::level::info, errMsg::location(),
                         dir, "ensure parents",
                         "is not a directory (st_mode is ",
                         static_cast<int>(stat->type),
                         ", perm: ", std::hex, stat->permissions,
                         ") but should be");
        }
    }
 
 
    void libssh::remove(const std::string& path, copyRequest::stateType& state) {
        auto stat = getUniquePtr(sftp_lstat(sftp, path.c_str()), sftp_attributes_free);
        if (stat == nullptr && sftp_get_error(sftp) == SSH_FX_NO_SUCH_FILE) {
            state |= copyRequest::stateBitType::vanished;
            return;
        }
        if (stat->type == SSH_FILEXFER_TYPE_DIRECTORY) {
            auto retval = sftp_rmdir(sftp, path.c_str());
            if (retval) {
                switch (sftp_get_error(sftp)) {
                    case SSH_FX_NO_SUCH_FILE:
                        state |= copyRequest::stateBitType::vanished;
                        return;
                    case SSH_FX_FILE_ALREADY_EXISTS:
                        errMsg::emit(errMsg::level::info, errMsg::location(),
                                     path, "remove", "directory not empty but should be removed");
                        state |= copyRequest::stateBitType::failed;
                        return;
                }
                throwcall::sftp::good0(retval, *this, "can't remove directory ", path);
            } else {
                state |= copyRequest::stateBitType::done;
                return;
            }
        } else {
            auto retval = sftp_unlink(sftp, path.c_str());
 
            if (retval && sftp_get_error(sftp) == SSH_FX_NO_SUCH_FILE) {
                state |= copyRequest::stateBitType::vanished;
            } else {
                throwcall::sftp::good0(retval, *this, "can't unlink ", path);
            }
            state |= copyRequest::stateBitType::done;
        }
    }
 
    bool libssh::renameSimple(const std::string& fromPath,
                              const std::string& toPath) {
        if (pathExists(toPath)) {
            auto retval = sftp_unlink(sftp, toPath.c_str());
            if (retval && sftp_get_error(sftp) == SSH_FX_NO_SUCH_FILE) {
                errMsg::emit(errMsg::level::notice, errMsg::location(),
                             toPath, "unlink in renameSimple",
                             "vanished");
            } else {
                throwcall::sftp::good0(retval, *this, "can't unlink in renameSimple ", toPath);
            }
        }
        auto retval = sftp_rename(sftp, fromPath.c_str(), toPath.c_str());
        if (retval && (sftp_get_error(sftp) == SSH_FX_NO_SUCH_FILE)) {
            return false;
        }
        throwcall::sftp::good0(retval, *this, "can't rename '", fromPath, "' to '", toPath, "'");
        return true;
    }
 
 
    base::renameRetvalType libssh::rename(const std::string& fromPath,
                                          const std::unique_ptr<const genericStat>& readInitialStat,
                                          const std::string& toPath,
                                          copyRequest::stateType& state) {
        auto fromPathStatBuf = getUniquePtr(sftp_stat(sftp, fromPath.c_str()),
                                            sftp_attributes_free);
        if (fromPathStatBuf == nullptr && sftp_get_error(sftp) == SSH_FX_NO_SUCH_FILE) {
            if (readInitialStat && readInitialStat->isDir()) { // no need to act, dirs are created on the fly
                state |= copyRequest::stateBitType::ignore;
                return base::renameRetvalType::ok;
            }
            return base::renameRetvalType::fileVanished;
        } else if (fromPathStatBuf == nullptr) { // curious error
            throwcall::sftp::badval(nullptr, nullptr, *this, "can't stat '", fromPath, "'");
            return base::renameRetvalType::cantHappen; // never reached
        }
        genericSftpStat fromPathStat(*fromPathStatBuf, getVfsStat(fromPath));
        if (!readInitialStat || (! readInitialStat->isDir() &&
                                 (fromPathStat.size != readInitialStat->size
                                  || !fromPathStat.isSameMtimeAs(*readInitialStat)))) {
            // the copy on fromPath is not up to date, create a new copy
            if (readInitialStat) {
                std::string initialTime;
                std::string finalTime;
                readInitialStat->getMtime(initialTime);
                fromPathStat.getMtime(finalTime);
                errMsg::emit(errMsg::level::info, errMsg::location(),
                             fromPath, "move"
                             "file (", toPath, ") changed unexpectedly ("
                             , initialTime, " -> ", finalTime, ", "
                             , readInitialStat->size, " -> ", fromPathStat.size
                             , ", doing fresh copy");
            } else {
                errMsg::emit(errMsg::level::info, errMsg::location(),
                             fromPath, "move",
                             "file (", toPath, ") has no initial stat, doing fresh copy");
            }
            return base::renameRetvalType::fileChanged;
        } else { // try to move the existing copy
            auto retval = sftp_rename(sftp, fromPath.c_str(), toPath.c_str());
 
            if (retval && (errno == ENOENT || errno == ENOTDIR)) {
                if (readInitialStat->isDir()) { // no need to act, dirs are created on the fly
                    state |= copyRequest::stateBitType::ignore;
                    return base::renameRetvalType::ok;
                }
                errMsg::emit(errMsg::level::info, errMsg::location(),
                             fromPath, "move",
                             "vanished unexpectedly, doing fresh copy");
                // file was probably not copied yet, try to copy it from the move target
                return base::renameRetvalType::fileVanished;
            } else {
                throwcall::good0(retval, "can't rename '", fromPath, "' to '", toPath, "'");
                state |= copyRequest::stateBitType::done;
                return base::renameRetvalType::ok;
            }
        }
    }
 
 
    void libssh::createSymlink(const std::vector<char>& target,
                               const std::string& path,
                               uid_t uid, gid_t gid) {
        {
            timerInst(sftp_symlink);
            auto retval = sftp_symlink(sftp, target.data(), path.c_str());
            if (retval != 0 && sftp_get_error(sftp) != SSH_FX_FILE_ALREADY_EXISTS) {
                throwcall::sftp::good0(retval, *this,  "can't create link at ", path);
            }
        }
        if (static_cast<std::make_signed<decltype(gid)>::type>(gid) != -1
                || static_cast<std::make_signed<decltype(uid)>::type>(uid) != -1) {
            timerInst(sftp_chown);
            throwcall::sftp::good0(sftp_chown(sftp, path.c_str(), uid, gid), *this,
                                   "can't set owner/group (", uid, ",", gid, ") on ", path);
        }
    }
 
 
    libssh::writerLibssh::writerLibssh(const std::string& aPath,
                                       libsshCommon& aHandler,
                                       bool mightAppend,
                                       size_t sourceSize,
                                       size_t readBlockSize,
                                       copyRequest::stateType &state,
                                       bool noWrite,
                                       std::unique_ptr<ioHandle::attrDataType> aAttrData,
                                       std::unique_ptr<acl::list> /*aAclData*/):
        libsshIoCommon(aPath, aHandler),
        writeInitialStat(nullptr),
        attrData(std::move(aAttrData)) {
        if (noWrite) {
            file = nullptr;
            blockSize = 1024 * 1024;
        } else {
            auto openMode = O_CREAT | O_TRUNC | O_WRONLY;
            state.clear(copyRequest::stateBitType::append); // clear any old append bits, may be left over from former attempts
            if (mightAppend) {
                writeInitialStat = sftp_stat(handler.sftp, path.c_str());
                if (writeInitialStat == nullptr) {
                    if (sftp_get_error(handler.sftp) != SSH_FX_NO_SUCH_FILE) {
                        throwcall::sftp::good0(1, handler, "can't stat destination file ", path);
                    }
                } else {
                    if (static_cast<size_t>(writeInitialStat->size) < sourceSize &&
                            sourceSize > readBlockSize) {
                        state |= copyRequest::stateBitType::append;
                        openMode = O_WRONLY;
                    }
                }
            }
            static timer::anchor a("sftp_open");
            timer::instanceUnscoped i(a);
            file = throwcall::sftp::badval(sftp_open(handler.sftp, path.c_str(), openMode, S_IWUSR),
                                           nullptr, handler,
                                           "can't open ", path, " for writing");
            i.stop();
            if (writeInitialStat == nullptr) {
                timerInst(sftp_fstat);
                writeInitialStat = throwcall::sftp::badval(sftp_fstat(file), nullptr,
                                   handler,
                                   "can't stat destination file ", path);
            }
            blockSize = 16 * 1024;
        }
    };
 
    void libssh::writerLibssh::closeAndRemoveBadCopy() {
        static timer::anchor a("sftp_close");
        timer::instanceUnscoped i(a);
        if (sftp_close(file) == SSH_ERROR) {
            errMsg::emit(errMsg::level::debug, errMsg::location(),
                         path, "close during unwind ",
                         "sftp error ", sftp_get_error(handler.sftp),
                         ", ssh error ", ssh_get_error(handler.session));
        }
        i.stop();
        errMsg::emit(errMsg::level::debug, errMsg::location(),
                     path, "unlink failed copy", "due to exception");
        timerInst(sftp_unlink);
        auto retval = sftp_unlink(handler.sftp, path.c_str());
        if (retval && sftp_get_error(handler.sftp) != SSH_FX_NO_SUCH_FILE) {
            errMsg::emit(errMsg::level::err, errMsg::location(),
                         path, "can't remove bad copy ",
                         "sftp error ", sftp_get_error(handler.sftp),
                         ", ssh error ", ssh_get_error(handler.session));
        }
    };
 
    libssh::writerLibssh::~writerLibssh() noexcept(false) {
        if (writeInitialStat != nullptr) {
            sftp_attributes_free(writeInitialStat);
        }
        if (file != nullptr) {
            if (isUnwinding()) {
                closeAndRemoveBadCopy();
                return;
            }
            if (attrData) {
                try {
                    attrData->set(this);
                } catch (const std::exception& e) {
                    errMsg::emit(errMsg::level::debug, errMsg::location(),
                                 path, "set attr at close ",
                                 e.what());
                    closeAndRemoveBadCopy();
                    throw;
                }
            }
            timerInst(sftp_close);
            throwcall::badval(sftp_close(file), SSH_ERROR, "can't close ", path);
        }
    };
 
 
    void libssh::writerLibssh::seek(size_t position) {
        throwcall::sftp::good0(sftp_seek64(file, position), handler,
                               "can't seek ", path, " to ", position);
    }
    size_t libssh::writerLibssh::getSize() const {
        return writeInitialStat->size;
    }
 
    void libssh::writerLibssh::writeBlock(const block& b) {
        if (b.isHole()) { // just write a hole...
            auto currentPosition = sftp_tell64(file); // cannot fail, in constrast to docs
            auto holeEnd  = currentPosition + b.size();
            throwcall::sftp::good0(sftp_seek64(file, holeEnd), handler,
                                   "can't seek to hole end (", holeEnd, ") in ", path);
            return;
        }
 
        size_t bytes_writen_so_far = 0;
 
        while (bytes_writen_so_far < b.size()) {
            auto count = b.size() - bytes_writen_so_far;
 
            if (count > blockSize) {
                count = blockSize;
            }
            writeRateLimit.wait();
            timerInst(sftp_write);
            auto bytes_writen = sftp_write(file, b.bufferAt(bytes_writen_so_far), count);
            if (bytes_writen < 0) {
                throwcall::sftp::good0(-1, handler,  "write failed on ", path);
            }
            writeRateLimit.update(bytes_writen);
            bytes_writen_so_far += bytes_writen;
        }
    }
 
 
    void libssh::writerLibssh::doAttributePreservations(const genericStat& readInitialStat) {
        if (preserve.timestamps) {
            struct timeval times[2];
            readInitialStat.getAtime(times[0]);
            readInitialStat.getMtime(times[1]);
            timerInst(sftp_utimes);
            throwcall::sftp::good0(sftp_utimes(handler.sftp, path.c_str(), times), handler,
                                   "can't set time stamp of ", path);
        }
 
        if (preserve.mode) {
            timerInst(sftp_chmod);
            throwcall::sftp::good0(sftp_chmod(handler.sftp, path.c_str(), readInitialStat.mode), handler,
                                   "can't set mode of ", path);
        } else {
            auto oldmask = umask(0);
            timerInst(sftp_chmod);
            throwcall::sftp::good0(sftp_chmod(handler.sftp, path.c_str(), modeBits & ~oldmask), handler,
                                   "can't set mode of ", path);
            umask(oldmask);
        }
 
        if (gid != -1 || uid != -1) {
            timerInst(sftp_chown);
            throwcall::sftp::good0(sftp_chown(handler.sftp, path.c_str(), uid, gid), handler,
                                   "can't set owner/group (", uid, ",", gid, ") of ", path);
        } else if (preserve.ownership) {
            timerInst(sftp_chown);
            throwcall::sftp::good0(sftp_chown(handler.sftp, path.c_str(),
                                              readInitialStat.ownerUid, readInitialStat.ownerGid),
                                   handler,
                                   "can't set owner/group (", readInitialStat.ownerUid, ",", readInitialStat.ownerGid, ") of ",
                                   path);
        }
    }
 
 
    void libssh::writerLibssh::sync() {
        // old versions of libssh do not have fsync...
        #if LIBSFTP_VERSION > 3
        throwcall::sftp::good0(sftp_fsync(file), handler.sftp,
                               "can't sync ", path);
        #endif
    }
 
    size_t libssh::getMaxNameLength(const std::string& dirPath) {
        auto stat = getVfsStat(dirPath);
        if (stat.f_namemax > 0) {
            return stat.f_namemax;
        } else {
            return 255; // best guess if no sane value forem system
        }
    }
}; // end namespace outputHandler