dirCount.cpp

Go to the documentation of this file.

#include <chrono>
#include <Options.h>
#include <scoped.h>
#include <throwcall.h>
#include <forward_list>
#include <fcntl.h>
#include <thread>
#include "timer.h"
#include <deque>
#include <map>
#include <set>
#include <mutex>
#include <condition_variable>
#include <stdexcept>
#include <thread>
#include <memory>
#include <iostream>
 
namespace MyWaitQueues {
    template <typename T> class simple {
      private:
        std::deque<std::unique_ptr<T>> queue;
        std::mutex queue_mutex;
      public:
        void enqueue(std::unique_ptr<T>& item) {
            std::unique_lock<decltype(queue_mutex)> lock(queue_mutex); //(NOLINT clang-analyzer-alpha.core.CastToStruct)
            queue.emplace_back(std::move(item));
        }
        template <class ... Types> void emplace(Types ... args) {
            std::unique_ptr<T> item(new T(args...));
            enqueue(item);
        }
        template <class ... Types> std::unique_ptr<T> dequeue() {
            std::unique_lock<decltype(queue_mutex)> lock(queue_mutex); //(NOLINT clang-analyzer-alpha.core.CastToStruct)
            if (queue.empty()) {
                return nullptr;
            }
            auto item = std::move(queue.front());
            queue.pop_front();
            return item;
        }
        decltype(queue.size()) size() const {
            return queue.size();
        }
        decltype(queue.empty()) empty() const {
            return queue.empty();
        }
    };
} // namespace MyWaitQueues
 
 
 
 
 
static options::single<bool> ignoreAccess('i', "ignoreAccess",
        "ignore subdirs we may not access");
static options::single<bool> quiet('q', "quiet",
                                   "be quiet, no prefix to output lines");
static options::single<bool> reportUnknowns('U',"reportUnknowns",
                                            "print a list of items with type DT_UNKNOWN");
 
class counterType {
  public:
    enum specials : unsigned char {
        denied = 253,
        noent = 254,
        notdir = 255
    };
    class itemDescriptor {
      public:
        const unsigned char index;
        const std::string name;
        const unsigned char optionChar;
        options::single<bool>* option;
        itemDescriptor(unsigned char aIndex,
                       const std::string& aName,
                       unsigned char aOptionChar):
            index(aIndex),
            name(aName),
            optionChar(aOptionChar) {
            option = new options::single<bool>(aOptionChar, aName,
                                               "print count of " + aName + " items");
        }
    };
  private:
    std::array<uint64_t, 256> counters;
    static const std::vector<itemDescriptor>& keyToName() {
        const static std::vector<itemDescriptor> kTN{
            {DT_BLK, "blockDevice", 'b'},
            {DT_CHR, "characterDevice", 'c'},
            {DT_DIR, "directory", 'd'},
            {DT_FIFO, "fifo", 'p'},
            {DT_LNK, "symlink", 'l'},
            {DT_REG, "file", 'f'},
            {DT_SOCK, "socket", 's'},
            {DT_UNKNOWN, "unknown", 'u'},
            {specials::denied, "accesDenied", 'D'},
            {specials::noent, "noSuchFile", 'M'},
            {specials::notdir, "notADirectory", 'N'}
        };
        return kTN;
    }
  public:
    counterType(): counters{0} {};
    static void init() {
        keyToName();
    }
    void count(unsigned char what) {
        counters[what]++;
    }
    void sum(counterType& other) {
        for (const auto& item : keyToName()) {
            other.counters[item.index] += counters[item.index];
        }
    }
    void print() {
        bool printAll(true);
        for (const auto& item : keyToName()) {
            if (*item.option) {
                printAll = false;
            }
        }
        for (const auto& item : keyToName()) {
            if (printAll || *item.option) {
                if (!quiet) {
                    std::cout << item.name << " ";
                }
                std::cout << counters[item.index] << "\n";
            }
        }
    }
};
 
typedef MyWaitQueues::simple<std::string> dirQueueType;
static options::single<unsigned> nThreads('n', "nThreads", "number of threads", 0);
 
static std::forward_list<std::string> unknownList;
static std::mutex unknownListMutex; 
 
class runnerType {
    counterType counter;
    unsigned int dirsDone;
    static std::mutex listMutex;
    static std::condition_variable cv;
    static std::atomic<unsigned int>nActiveWorkers;
  public:
    static std::forward_list<runnerType>& runners() {
        static std::forward_list<runnerType> list;
        return list;
    }
    static unsigned int nWorkers;
    std::thread worker;
    runnerType(dirQueueType& dirQueue): dirsDone(0) {
        worker = std::thread(&runnerType::process, this, std::ref(dirQueue));
    }
  public:
    static void newWorker(dirQueueType& dirQueue) {
        std::unique_lock<decltype(listMutex)> lock(listMutex);
        if (nWorkers < nThreads) {
            nWorkers++;
            nActiveWorkers++;
            runners().emplace_front(dirQueue);
        }
    }
    static void finish(counterType& sumCounter, bool verbose) {
        std::unique_lock<decltype(listMutex)> lock(listMutex);
        cv.wait(lock, [] {return nActiveWorkers == 0;});
        for (auto& runner : runners()) {
            runner.worker.join();
            runner.sum(sumCounter);
        }
        if (verbose) {
            unsigned int realizedThreads(0);
            unsigned int unusedThreads(0);
            for (auto& runner : runners()) {
                realizedThreads++;
                if (runner.getDirsDone() == 0) {
                    unusedThreads++;
                }
            }
            std::cerr << "of " << realizedThreads << " threads " << unusedThreads << " were not used\n";
        }
    }
 
 
 
    void sum(counterType& other) {
        counter.sum(other);
    }
    unsigned int getDirsDone() const {
        return dirsDone;
    }
    void process(dirQueueType& dirQueue) {
        while (auto path = dirQueue.dequeue()) {
            dirsDone++;
            static timer::anchor a("opendir");
            timer::instanceUnscoped i(a);
            auto result = opendir(path->c_str());
            i.stop();
            if (result == nullptr) {
                if (errno == EACCES && ignoreAccess) {
                    counter.count(counterType::specials::denied);
                    continue;
                } else if (errno == ENOENT) {
                    counter.count(counterType::specials::noent);
                    continue;
                } else if (errno == ENOTDIR) {
                    counter.count(counterType::specials::notdir);
                    continue;
                }
                throwcall::badval(result, nullptr, "can't open directory ", *path);
            }
            scoped::Dir dir(*path, result);
            static timer::anchor b("readdir");
            timer::instanceUnscoped I(b);
            while (auto entry = readdir(dir)) {
                I.stop();
                if (entry->d_name[entry->d_name[0] != '.' ? 0 : entry->d_name[1] != '.' ? 1 : 2] == '\0') {
                    I.restart();
                    continue; // skip . .. and empty strings
                }
                counter.count(entry->d_type);
                if (entry->d_type == DT_DIR) {
                    std::string subdir(*path);
                    subdir += "/";
                    subdir += entry->d_name;
                    dirQueue.emplace(subdir);
                    newWorker(dirQueue);
                } else if (entry->d_type == DT_UNKNOWN && reportUnknowns) {
                    std::string item(*path);
                    item += "/";
                    item += entry->d_name;
                    std::unique_lock<decltype(unknownListMutex)> lock(unknownListMutex); //(NOLINT clang-analyzer-alpha.core.CastToStruct)
                    unknownList.emplace_front(item);
                }
                I.restart();
            }
        }
        std::unique_lock<decltype(listMutex)> lock;
        --nActiveWorkers;
        cv.notify_one();
    }
};
 
unsigned int runnerType::nWorkers(0);
std::mutex runnerType::listMutex;
std::condition_variable runnerType::cv;
std::atomic<unsigned int> runnerType::nActiveWorkers(0);
 
 
void handleDir(const std::string& path, counterType& counter) {
    static timer::anchor a("opendir");
    timer::instanceUnscoped i(a);
    auto result = opendir(path.c_str());
    i.stop();
    if (result == nullptr) {
        if (errno == EACCES && ignoreAccess) {
            counter.count(counterType::specials::denied);
            return;
        } else if (errno == ENOENT) {
            counter.count(counterType::specials::noent);
            return;
        } else if (errno == ENOTDIR) {
            counter.count(counterType::specials::notdir);
            return;
        }
        throwcall::badval(result, nullptr, "can't open directory ", path);
    }
    scoped::Dir dir(path, result);
    std::forward_list<std::string> subdirs;
    static timer::anchor b("readdir");
    timer::instanceUnscoped I(b);
    while (auto entry = readdir(dir)) {
        I.stop();
        if (entry->d_name[entry->d_name[0] != '.' ? 0 : entry->d_name[1] != '.' ? 1 : 2] == '\0') {
            I.restart();
            continue; // skip . .. and empty strings
        }
        counter.count(entry->d_type);
        if (entry->d_type == DT_DIR) {
            subdirs.emplace_front(entry->d_name);
        } else if (entry->d_type == DT_UNKNOWN && reportUnknowns) {
            std::string item(path);
            item += "/";
            item += entry->d_name;
            unknownList.emplace_front(item);
        }
        I.restart();
    }
    for (const auto& subdir : subdirs) {
        handleDir(path + "/" + subdir, counter);
    }
 
}
 
int main(int argc, const char *argv[]) {
    options::parser parser("dirCount", "", {});
    options::positional<options::container<std::string>>paths(10, "paths",
            "paths to consider");
    options::single<bool> timing('t', "timing", "show timing info");
    counterType::init();
    parser.fParse(argc, argv);
 
    counterType counter;
    if (nThreads == 1) {
        for (auto path : paths) {
            handleDir(path, counter);
        }
    } else {
        if (nThreads == 0) {
            nThreads = sysconf(_SC_NPROCESSORS_ONLN);
        }
        dirQueueType dirQueue;
        for (const auto& path : paths) {
            dirQueue.emplace(path);
        }
        for (unsigned int i = 0; i < paths.size(); i++) {
            runnerType::newWorker(dirQueue);
        }
        runnerType::finish(counter, timing);
    }
    counter.print();
    if (timing) {
        timer::anchor::print(std::cerr, "");
    }
    if (reportUnknowns) {
        std::cout << "List of items of type DT_UNKNOWN:\n";
        for (const auto& item: unknownList) {
            std::cout << item << "\n";
        }
    }
    return 0;
}