root/hydranode/hncore/imp_mldonkey/imp_mldonkey.cpp

/*
 *  Copyright (C) 2006 Alo Sarv <madcat_@users.sourceforge.net>
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

#include <boost/algorithm/string/replace.hpp>
#include <boost/algorithm/string/trim.hpp>
#include <boost/algorithm/string/split.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/spirit.hpp>
#include <boost/spirit/actor.hpp>
#include <hnbase/hash.h>
#include <hnbase/range.h>
#include <hnbase/rangelist.h>
#include <hncore/metadata.h>
#include <hncore/partdata.h>
#include <hncore/sharedfile.h>
#include <hncore/fileslist.h>
#include <hncore/modules.h>
#include <iostream>
#include <fstream>
#include <vector>
#include <map>
#include <set>

struct FileInfo {
        std::string m_network;
        uint64_t    m_size;
        uint8_t     m_priority;
        std::string m_state;
        std::string m_fileName;
        uint32_t    m_age;
        bool        m_primary;
        std::string m_swarmer;
        uint32_t    m_mTime;
        std::string m_chunks;
        uint64_t    m_downloaded;
        std::string m_tempName;
        std::vector<uint32_t>        m_chunksAge;
        RangeList64                  m_completed;
        RangeList64                  m_verified;
        ED2KHashSet                  m_hashSet;
        std::vector<std::string>     m_chunkHashes;
        std::vector<std::string>     m_fileNames;
        FileInfo();
        friend std::ostream& operator<<(std::ostream &o, const FileInfo &i);
};

FileInfo::FileInfo() : m_size(), m_priority(), m_age(), m_primary(), m_mTime(),
m_downloaded() {}

std::ostream& operator<<(std::ostream &o, const FileInfo &i) {
        o << "  Network:    " << i.m_network    << std::endl;
        o << "  Size:       " << i.m_size       << std::endl;
        o << "  Priority:   " << i.m_priority   << std::endl;
        o << "  State:      " << i.m_state      << std::endl;
        o << "  FileName:   " << i.m_fileName   << std::endl;
        o << "  Age:        " << i.m_age        << std::endl;
        o << "  Primary:    " << i.m_primary    << std::endl;
        o << "  Swarmer:    " << i.m_swarmer    << std::endl;
        o << "  ModTime:    " << i.m_mTime      << std::endl;
        o << "  Chunks:     " << i.m_chunks     << std::endl;
        o << "  Downloaded: " << i.m_downloaded << std::endl;
        o << "  TempName:   " << i.m_tempName   << std::endl;
        return o;
}

class Imp_MlDonkey : public ModuleBase {
        DECLARE_MODULE(Imp_MlDonkey, "imp_mldonkey");
public:
        virtual bool onInit();
        virtual int onExit();
private:
        void import(boost::filesystem::path from);
};

IMPLEMENT_MODULE(Imp_MlDonkey);

bool Imp_MlDonkey::onInit() {
        FilesList::instance().import.connect(
                boost::bind(&Imp_MlDonkey::import, this, _1)
        );
        return true;
}

int Imp_MlDonkey::onExit() {
        return 0;
}

void Imp_MlDonkey::import(boost::filesystem::path from) {
        using namespace boost::algorithm;
        using namespace boost::spirit;

        std::ifstream ifs(
                (from / "files.ini").native_directory_string().c_str(), 
                std::ios::in
        );
        std::string tmp;
        bool inFiles(false), inFile(false);
        std::string fileData;
        std::vector<std::string> files;

        // First round of processing - locate the 'files' section, and separate
        // each file into separate string.
        while (getline(ifs, tmp)) {
                trim(tmp);
                if (!inFiles && tmp == "files = [") {
                        inFiles = true;
                } else if (inFiles && !inFile && tmp.find("{") == 0) {
                        inFile = true;
                        fileData += tmp.substr(1) + "\n";
                } else if (inFiles && inFile && tmp == "};") {
                        files.push_back(fileData);
                        fileData.clear();
                        inFile = false;
                } else if (inFiles && inFile && tmp == "};]") {
                        files.push_back(fileData);
                        fileData.clear();
                        inFile = false;
                        inFiles = false;                        
                } else if (inFiles && inFile) {
                        fileData += tmp + "\n";
                }
        }
        std::cerr << "Found " << files.size() << " temp files." << std::endl;

        // Second round of processing - for each file, read all the keys
        // (note that some keys may cross multiple lines), first copy then into
        // key/val map of fileKeys, and near the end of section, construct
        // FileInfo object and assign proper keys to proper variables (casting
        // as needed).
        std::vector<FileInfo> importList;
        for (size_t i = 0; i < files.size(); ++i) {
                std::map<std::string, std::string> fileKeys;
                std::vector<std::string> lines;
                split(lines, files[i], is_any_of("\n"));
                std::string blockData;
                std::string curKey, curValue;
                bool inBlock(false);
                for (size_t j = 0; j < lines.size(); ++j) {
                        if (!lines[j].size()) {
                                continue;
                        }
                        if (!inBlock && iends_with(lines[j], "[")) {
                                std::vector<std::string> keyVal;
                                split(keyVal, lines[j], is_any_of("="));
                                if (!keyVal.size()) {
                                        std::cerr << "Parse failure on line '";
                                        std::cerr << lines[j];
                                        std::cerr << "' (expected '=' token)";
                                        std::cerr << std::endl;
                                }
                                curKey = keyVal[0];
                                boost::algorithm::trim(curKey);
                                inBlock = true;
                        } else if (inBlock && iends_with(lines[j], "]")) {
                                inBlock = false;
                                blockData += lines[j].substr(
                                        0, lines[j].size() - 1
                                );
                                fileKeys[curKey] = blockData;
                                blockData.clear();
                        } else if (inBlock) {
                                blockData += lines[j];
                        } else {
                                std::vector<std::string> keyVal;
                                split(keyVal, lines[j], is_any_of("="));
                                if (keyVal.size() != 2) {
                                        std::cerr << "Parse failure on line '";
                                        std::cerr << lines[j];
                                        std::cerr << "' (expected '=' token)";
                                        std::cerr << std::endl;
                                } else {
                                        curKey = keyVal[0];
                                        curValue = keyVal[1];
                                        trim(curKey);
                                        trim(curValue);
                                        fileKeys[curKey] = curValue;
                                }
                        }
                }
                FileInfo f;
                f.m_network = fileKeys["file_network"];
                try {
                        f.m_size = boost::lexical_cast<uint64_t>(
                                fileKeys["file_size"]
                        );
                } catch (...) {
                        std::cerr << "Failed to cast " << fileKeys["file_size"];
                        std::cerr << "to integer (file_size key).\n";
                }
                try {
                        f.m_priority = boost::lexical_cast<uint8_t>(
                                fileKeys["file_priority"]
                        );
                } catch (...) {
                        std::cerr << "Failed to cast ";
                        std::cerr << fileKeys["file_priority"];
                        std::cerr << "to integer (file_priority key).\n";
                }
                f.m_state = fileKeys["file_state"];
                f.m_fileName = fileKeys["file_filename"];
                // filenames are sometimes quoted
                trim_if(f.m_fileName, is_any_of("\"")); 
                try {
                        f.m_age = boost::lexical_cast<uint32_t>(
                                fileKeys["file_age"]
                        );
                } catch (...) {
                        std::cerr << "Failed to cast " << fileKeys["file_age"];
                        std::cerr << "to integer (file_age key).\n";
                }
                f.m_primary = fileKeys["file_primary"] == "true" ? true : false;
                f.m_swarmer = fileKeys["file_swarmer"];
                trim_if(f.m_swarmer, is_any_of("\""));
                try {
                        if (iends_with(fileKeys["file_mtime"], ".")) {
                                fileKeys["file_mtime"] = fileKeys["file_mtime"]
                                .substr(0, fileKeys["file_mtime"].size() - 1);
                        }
                        f.m_mTime = boost::lexical_cast<uint32_t>(
                                fileKeys["file_mtime"]
                        );
                } catch (...) {
                        std::cerr << "Failed to cast ";
                        std::cerr << fileKeys["file_mtime"];
                        std::cerr << "to integer (file_mtime key).\n";
                }
                f.m_chunks = fileKeys["file_chunks"];
                try {
                        f.m_downloaded = boost::lexical_cast<uint32_t>(
                                fileKeys["file_downloaded"]
                        );
                } catch (...) {
                        std::cerr << "Failed to cast ";
                        std::cerr << fileKeys["file_downloaded"];
                        std::cerr << "to integer (file_downloaded key).\n";
                }
                std::string tmp = fileKeys["file_md4"];
                trim_if(tmp, is_any_of("\""));
                f.m_hashSet.setFileHash(Utils::encode(tmp));
                f.m_tempName = fileKeys["file_diskname"];

                // makes parsing easier
                fileKeys["file_present_chunks"] += ";";

                std::vector<uint64_t> begins, ends;
                parse_info<> ret = parse(
                        fileKeys["file_present_chunks"].c_str(),
                        *('(' >> uint_p[push_back_a(begins)] >> ',' 
                          >> uint_p[push_back_a(ends)] >> ')' >> ';'),
                        space_p
                );
                if (ret.full) {
                        for (size_t i = 0; i < begins.size(); ++i) {
                                f.m_completed.merge(begins[i], ends[i]);
                        }
                } else if (fileKeys["file_present_chunks"].size() > 3) {
                        std::cerr << "Parse failure (file_present_chunks key)";
                        std::cerr << std::endl;
                        std::cerr << fileKeys["file_present_chunks"];
                        std::cerr << std::endl;
                }

                std::vector<std::string> chunkHashes;
                split(chunkHashes, fileKeys["file_md4s"], is_any_of(";"));
                for (size_t i = 0; i < chunkHashes.size(); ++i) {
                        trim_if(chunkHashes[i], is_any_of("\""));
                        if (!chunkHashes[i].size()) {
                                continue;
                        }
                        f.m_hashSet.addChunkHash(Utils::encode(chunkHashes[i]));
                }

                replace_all(f.m_swarmer, "\\\\", "\\");
                replace_all(f.m_chunks, "\"", "");

                for (size_t i = 0; i < f.m_chunks.size(); ++i) {
                        if (f.m_chunks[i] == 3) {
                                uint64_t begin = i * (9500 * 1024);
                                uint64_t end = i + 1 * (9500 * 1024) - 1;
                                if (end > f.m_size - 1) {
                                        end = f.m_size - 1;
                                }
                                f.m_verified.merge(begin, end);
                        }
                }

                // only donkey downloads supported currently
                if (f.m_network == "Donkey") {
                        importList.push_back(f);
                }
        }

        using namespace boost::filesystem;

        for (size_t i = 0; i < importList.size(); ++i) {
                try {
                        std::cerr << "#" << i << ": ";
                        std::cerr << importList[i].m_fileName << std::endl;
                        std::cerr << "Size: " << importList[i].m_size 
                                << std::endl;

                        replace_all(importList[i].m_swarmer, "\\", "/");

                        MetaData *md = new MetaData(importList[i].m_size);
                        md->addHashSet(
                                new ED2KHashSet(importList[i].m_hashSet)
                        );
                        md->addFileName(importList[i].m_fileName);

                        PartData *pd = new PartData(
                                from / path(importList[i].m_swarmer, native), 
                                md, importList[i].m_completed,
                                importList[i].m_verified
                        );

                        SharedFile *sf = new SharedFile(pd, md);

                        FilesList::instance().push(sf);
                } catch (std::exception &e) {
                        logError(e.what());
                }
                FilesList::instance().addTempDir(
                        (from / path(importList[i].m_swarmer, native))
                        .branch_path().string(), false
                );
        }
}