Changeset 2738

Timestamp:

03/02/06 00:41:31 (3 years ago)

Author:

madcat

Message:

Support for global searching.

Files:

• hydranode/hncore/ed2k/clientlist.cpp (modified) (1 diff)
• hydranode/hncore/ed2k/ed2ksearch.cpp (modified) (2 diffs)
• hydranode/hncore/ed2k/ed2ksearch.h (modified) (1 diff)
• hydranode/hncore/ed2k/opcodes.h (modified) (1 diff)
• hydranode/hncore/ed2k/packets.cpp (modified) (3 diffs)
• hydranode/hncore/ed2k/packets.h (modified) (2 diffs)
• hydranode/hncore/ed2k/server.cpp (modified) (3 diffs)
• hydranode/hncore/ed2k/server.h (modified) (5 diffs)
• hydranode/hncore/ed2k/serverlist.cpp (modified) (7 diffs)
• hydranode/hncore/ed2k/serverlist.h (modified) (3 diffs)
• hydranode/hncore/hnsh/shellcommands.cpp (modified) (14 diffs)
• hydranode/hncore/hnsh/shellcommands.h (modified) (3 diffs)
• hydranode/hncore/search.cpp (modified) (5 diffs)
• hydranode/hncore/search.h (modified) (10 diffs)

hydranode/hncore/ed2k/clientlist.cpp

@@ -813 +813 @@
-                        ServerList::instance().handleGlobStatRes(packet, from);
-                        break;
+                case OP_GLOBSEARCHRES:
+                        ServerList::instance().handleGlobSearchRes(
+                                packet, from
+                        );
+                        break;
-                default:
-                        logDebug(

hydranode/hncore/ed2k/ed2ksearch.cpp

@@ -29 +29 @@
-#include <hncore/fileslist.h>
-#include <hncore/sharedfile.h>
+#include <boost/lexical_cast.hpp>
-
-namespace Donkey {
@@ -75 +76 @@
-}
-
+std::string ED2KSearchResult::identifier() const {
+        return m_hash.toString() + boost::lexical_cast<std::string>(getSize());
+}
+
-} // end namespace Donkey

hydranode/hncore/ed2k/ed2ksearch.h

@@ -52 +52 @@
-         */
-        virtual void download();
+
+        /**
+         * @returns unique identifier (hash + size combo) for this result
+         */
+        virtual std::string identifier() const;
-private:
-        Hash<ED2KHash> m_hash;

hydranode/hncore/ed2k/opcodes.h

@@ -206 +206 @@
-        OP_GLOBGETSOURCES2  = 0x94,  //!< cnt*[<hash>hash<u32>size]
-        OP_GLOBFOUNDSOURCES = 0x9b,  //!< <hash>hash<u8>cnt*[<u32>id<u16>port]
-
+
+
-        OP_GLOBSTATREQ      = 0x96,  //!< <u32>challenge
-        //! <u32>challenge<u32>users<u32>files<u32>maxusers<u32>softlimit

hydranode/hncore/ed2k/packets.cpp

@@ -397 +397 @@
-        Utils::StopWatch stopwatch;
-        uint32_t cnt2 = count; // used for later debug msg
+
-        while (count-- && i) {
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-        }
-
@@ -474 +408 @@
-                ) % cnt2 % stopwatch
-        );
+}
+
+// protected default constructor for usage by GlobSearchRes class
+SearchResult::SearchResult() {}
+
+void SearchResult::readResult(std::istream &i) {
+        Hash<ED2KHash> h(Utils::getVal<std::string>(i, 16));
+        (void)Utils::getVal<uint32_t>(i); // id - ignored
+        (void)Utils::getVal<uint16_t>(i); // port - ignored
+        uint32_t tagCount = Utils::getVal<uint32_t>(i);
+        std::string name;
+        uint32_t size = 0;
+        uint32_t sources = 0;
+        uint32_t completesrc = 0;
+        std::string codec;
+        uint32_t bitrate = 0;
+        uint32_t len = 0;
+        uint32_t lastSeen = 0;
+        uint8_t  rating = 0;
+        while (tagCount-- && i) {
+                Tag t(i);
+                switch (t.getOpcode()) {
+                        case CT_FILENAME:
+                                name = t.getStr();
+                                break;
+                        case CT_FILESIZE:
+                                size = t.getInt();
+                                break;
+                        case CT_SOURCES:
+                                sources = t.getInt();
+                                break;
+                        case CT_COMPLSRC:
+                                completesrc = t.getInt();
+                                break;
+                        case CT_MEDIA_CODEC:
+                                codec = t.getStr();
+                                break;
+                        case CT_MEDIA_BITRATE:
+                                bitrate = t.getInt();
+                                break;
+                        case CT_MEDIA_LENGTH:
+                                len = t.getInt();
+                                break;
+                        case CT_LASTSEENCOMPL:
+                                lastSeen = t.getInt();
+                                break;
+                        case CT_FILERATING:
+                                rating = t.getInt();
+                                break;
+                        default:
+                                if (t.getName() == "bitrate") {
+                                        bitrate = t.getInt();
+                                } else if (t.getName() == "length") {
+                                        /**
+                                         * TODO: Convert str-tag LEN
+                                         *       into integer-len
+                                         */
+                                }
+                                warnUnHandled("SearchResult", t);
+                                break;
+                }
+        }
+        boost::shared_ptr<ED2KSearchResult> f;
+
+        // Note - ignoring ID/Port values.
+        f.reset(new ED2KSearchResult(h, name, size));
+        f->addSources(sources);
+        f->addComplete(completesrc);
+        f->addRating(rating);
+        if (codec.size() || bitrate || len) {
+                f->getStrd().reset(new StreamData(codec, bitrate, len));
+        }
+
+        m_results.push_back(f);
-}
-
@@ -579 +587 @@
-        } catch (std::exception &) {} // safe to ignore
-}
+
+// GlobSearchReq class
+// -------------------
+GlobSearchReq::GlobSearchReq(SearchPtr data, uint8_t proto)
+: Search(data, proto) {}
+
+GlobSearchReq::operator std::string() {
+        std::string tmp(*static_cast<Search*>(this));
+        tmp.erase(1, 4); // no size field in udp packets
+        tmp[1] = OP_GLOBSEARCHREQ;
+        return tmp; // makePacket is called by Search class already
+}
+
+// GlobSearchRes class
+// -------------------
+GlobSearchRes::GlobSearchRes(std::istream &i) : SearchResult() {
+        while (i) {
+                readResult(i);
+                if (i.peek() == PR_ED2K) {
+                        (void)Utils::getVal<uint8_t>(i);
+                } else {
+                        i.seekg(-1, std::ios::cur);
+                        break;
+                }
+                if (i.peek() == OP_GLOBSEARCHRES) {
+                        (void)Utils::getVal<uint8_t>(i);
+                } else {
+                        i.seekg(-2, std::ios::cur);
+                        break;
+                }
+        }
+        if (!i.eof()) {
+                logDebug(boost::format(
+                        "Extra bytes at the end of GlobSearchRes packet: %s"
+                ) % Utils::hexDump(
+                        dynamic_cast<std::istringstream&>(i).str().substr(
+                                i.tellg()
+                        )
+                ));
+        }
+}
+
-
-                      /*************************/

hydranode/hncore/ed2k/packets.h

@@ -323 +323 @@
-                return m_results.at(num);
-        }
+protected:
+        SearchResult();
+
+        //! Reads one result from stream
+        void readResult(std::istream &i);
-private:
-        std::vector<boost::shared_ptr<ED2KSearchResult> > m_results;
@@ -462 +467 @@
-        uint32_t m_udpFlags;    //!< UDP flags
-        uint32_t m_lowIdUsers;  //!< Number of online LowId users
+};
+
+/**
+ * Global search request, sent via UDP socket to servers; format is
+ * same as with Search packet, except for different opcode.
+ *
+ * Usage: Client -> Server (UDP)
+ */
+class GlobSearchReq : public Search {
+public:
+        GlobSearchReq(SearchPtr data, uint8_t proto = PR_ED2K);
+        operator std::string();
+};
+
+/**
+ * Response to GlobSearchReq packet, sent by servers via UDP; format is
+ * same as SearchResult packet, except for different opcode.
+ */
+class GlobSearchRes : public SearchResult {
+public:
+        GlobSearchRes(std::istream &i);
-};
-

hydranode/hncore/ed2k/server.cpp

@@ -38 +38 @@
-m_addr(addr), m_failedCount(), m_preference(), m_ping(), m_lastPing(),
-m_maxUsers(), m_softLimit(), m_hardLimit(), m_udpFlags(), m_users(), m_files(),
-
+
+
-        // Sanity checking
-        CHECK_THROW_MSG(addr.getAddr(), "Server IP may not be null.");
@@ -44 +45 @@
-
-        setName(m_addr.getStr());
+
+        // mark all servers udp-search capable initially, so when starting
+        // with empty server list, it's still possible to perform searches
+        m_udpFlags &= FL_GETFILES; 
-}
-
@@ -53 +58 @@
-m_failedCount(), m_preference(), m_ping(), m_lastPing(), m_maxUsers(),
-m_softLimit(), m_hardLimit(), m_udpFlags(), m_users(), m_files(),
-
+
+
-        m_addr.setAddr(Utils::getVal<uint32_t>(i));
-        m_addr.setPort(Utils::getVal<uint16_t>(i));

hydranode/hncore/ed2k/server.h

@@ -28 +28 @@
-#include <hnbase/ipv4addr.h>
-#include <hnbase/object.h>
+#include <hncore/fwd.h>
-#include <hncore/ed2k/fwd.h>
-
@@ -84 +85 @@
-        bool        pingInProgress()        const { return m_pingInProgress; }
-        uint32_t    getChallenge()          const { return m_challenge;      }
+        SearchPtr   currentSearch()         const { return m_globSearch;     }
-
-        //! ED2K Server UDP port is always TCP + 4
@@ -89 +91 @@
-        bool supportsGetSources() const { return m_udpFlags & FL_GETSOURCES; }
-        bool supportsGetFiles()   const { return m_udpFlags & FL_GETFILES;   }
+        bool supportsUdpSearch()  const { return m_udpFlags & FL_GETFILES;   }
-        //@}
-
@@ -116 +119 @@
-
-        void setLastUdpQuery(uint64_t time)   { m_lastUdpQuery = time; }
-
-
+
+
+
-        //@}
-private:
@@ -158 +162 @@
-        std::vector<uint16_t> m_auxPorts; //!< Auxiliary ports
-
-
-
-
+
+
+
+
-        //@}
-

hydranode/hncore/ed2k/serverlist.cpp

@@ -72 +72 @@
-};
-
+//! Server UDP Addr extractor functor
+struct ServerUAddrExtractor {
+        typedef IPV4Address result_type;
+        result_type operator()(const Server *const s) const {
+                return IPV4Address(s->getAddr().getIp(), s->getUdpPort());
+        }
+};
+
-//! last udp query time extractor functor
-struct QueryTimeExtractor {
@@ -87 +95 @@
-        boost::multi_index::ordered_non_unique<ServerAddrExtractor>,
-        boost::multi_index::ordered_non_unique<ServerNameExtractor>,
-
+
+
-> {};
-struct MIServerList : boost::multi_index_container<
@@ -99 +108 @@
-        QTIter(const T &t) : MIServerList::nth_index<3>::type::iterator(t) {}
-};
+typedef MIServerList::nth_index<4>::type::iterator UAddrIter;
-
-} // namespace Detail
@@ -455 +465 @@
-        logTrace(TRACE, "[ed2k] ServerList::performSearch");
-
-
-
-
-
-
-
-
-
-
+
+
-                *m_serverSocket << ED2KPacket::Search(search);
-                m_curSearch = search;
-        }
+
+        // global search
+        std::vector<Server*> tmp;
+        std::copy(m_list->begin(), m_list->end(), std::back_inserter(tmp));
+        std::random_shuffle(tmp.begin(), tmp.end());
+        boost::shared_ptr<std::list<Server*> > list(new std::list<Server*>);
+        std::copy(tmp.begin(), tmp.end(), std::back_inserter(*list));
+        Utils::timedCallback(
+                boost::bind(&ServerList::doGlobSearch, this, search, list), 0
+        );
+        std::string keywords;
+        for (uint32_t i = 0; i < search->getTermCount(); ++i) {
+                keywords += search->getTerm(i);
+                if (i + 1 < search->getTermCount()) {
+                        keywords += " ";
+                }
+        }
+        logMsg(
+                boost::format(
+                        "eDonkey2000: Search started with keywords '%s'."
+                ) % keywords
+        );
-}
-
@@ -784 +809 @@
-                ) % e.what()
-        );
-
-
+
-
-void ServerList::handleGlobStatRes(std::istringstream &i, IPV4Address from) try{
@@ -849 +873 @@
-                ) % e.what()
-        );
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-void ServerList::queryNextServer() {
@@ -1165 +1223 @@
-}
-
+void ServerList::doGlobSearch(
+        SearchPtr search, boost::shared_ptr<std::list<Server*> > srv
+) {
+        CHECK_RET(srv);
+        if (!search->isRunning() || !m_list->size() || !srv->size()) {
+                return;
+        }
+        uint32_t callback = 500; // default 500ms delay
+        size_t loopCnt = srv->size();
+
+        while (srv->size() && loopCnt--) {
+                if (m_list->find(srv->front()) == m_list->end()) {
+                        srv->pop_front();
+                } else if (srv->front()->currentSearch()) {
+                        srv->push_back(srv->front());
+                        srv->pop_front();
+                } else if (!srv->front()->supportsUdpSearch()) {
+                        srv->pop_front();
+                }
+        }
+        if (srv->size() && srv->front()->currentSearch()) {
+                logTrace(TRACE,
+                        "No valid servers found for global search, "
+                        "retrying in 5 seconds."
+                );
+                callback = 5000;
+        } else if (srv->size()) {
+                ED2KPacket::GlobSearchReq packet(search);
+                IPV4Address to(
+                        srv->front()->getAddr().getIp(), 
+                        srv->front()->getUdpPort()
+                );
+                Client::getUdpSocket()->send(packet, to);
+                srv->front()->setCurrentSearch(search);
+                Utils::timedCallback(
+                        boost::bind(
+                                &Server::setCurrentSearch,
+                                srv->front(), SearchPtr()
+                        ), 10000    // 10sec timeout
+                );
+                srv->pop_front();
+                logTrace(TRACE,
+                        boost::format("Sent global search request to %s") % to
+                );
+        }
+        if (srv->size()) {
+                Utils::timedCallback(
+                        boost::bind(
+                                &ServerList::doGlobSearch, this,
+                                search, srv
+                        ), callback
+                );
+        } else {
+                std::string keywords;
+                for (uint32_t i = 0; i < search->getTermCount(); ++i) {
+                        keywords += search->getTerm(i);
+                        if (i + 1 < search->getTermCount()) {
+                                keywords += " ";
+                        }
+                }
+                logMsg(
+                        boost::format(
+                                "eDonkey2000: Global search "
+                                "with keywords '%s' ended."
+                        ) % keywords
+                );
+        }
+}
+
-} // end namespace Donkey

hydranode/hncore/ed2k/serverlist.h

@@ -245 +245 @@
-         */
-        void onUdpData(ED2KUDPSocket *sock, SocketEvent evt);
+
+        /**
+         * Sends global UDP search query to next server in list.
+         */
+        void doGlobSearch(
+                SearchPtr search,
+                boost::shared_ptr<std::list<Detail::Server*> > srv
+        );
-public:
-        //! @name Packet handlers
@@ -256 +264 @@
-        void onPacket(const ED2KPacket::CallbackReq &p);
-        void onPacket(const ED2KPacket::FoundSources  &p);
-        //@}
-
-        /**
@@ -273 +280 @@
-         */
-        void handleGlobStatRes(std::istringstream &i, IPV4Address from);
+
+        /**
+         * Handles GlobSearchRes packet from UDP servers
+         *
+         * @param i      Input stream to read packet from
+         * @param from   The server that sent this data
+         */
+        void handleGlobSearchRes(std::istringstream &i, IPV4Address from);
+        //@}
-
-        //! @name Various operations

hydranode/hncore/hnsh/shellcommands.cpp

@@ -85 +85 @@
-        m_commands["search"]  = bind(&ShellCommands::cmdSearch, this, _1);
-        m_commands["vr"]      = bind(&ShellCommands::cmdViewResults, this, _1);
+        m_commands["ss"]      = bind(&ShellCommands::cmdStopSearch, this, _1);
-        m_commands["download"]  = bind(&ShellCommands::cmdDownload, this, _1);
-        m_commands["vd"]     = bind(&ShellCommands::cmdListDownloads, this, _1);
@@ -99 +100 @@
-        m_commands["hs"]   = bind(&ShellCommands::cmdShowHasherStats, this, _1);
-        m_commands["history"] = bind(&ShellCommands::cmdShowHistory, this, _1);
-s"] 
+cs"]
-        m_commands["ifs"]     = bind(&ShellCommands::cmdFilterStats, this, _1);
-#if !defined(NDEBUG) && !defined(NTRACE)
@@ -426 +427 @@
-        }
-
-
+
+
+
+
-
-        // Parse arguments
@@ -451 +455 @@
-                                boost::format("Adding search term '%s'.") % *i
-                        );
-s
+m_currentS
-                }
-        }
@@ -460 +464 @@
-                if ((*i).first == "minsize") {
-                        uint64_t mz = lexical_cast<uint64_t>((*i).second);
-s
+m_currentS
-                } else if ((*i).first == "maxsize") {
-                        uint64_t mz = lexical_cast<uint64_t>((*i).second);
-s
+m_currentS
-                } else if ((*i).first == "type") {
-s
+m_currentS
-                } else {
-                        boost::format fmt("Unknown argument: '%s'");
@@ -483 +487 @@
-                        (*i).first
-                );
+                m_currentSearch.reset();
-                return false;
-        }
-
-        // Parsing is done, check if everything we need is in place
-s
+m_currentS
-                setError("please specify a term to search for");
+                m_currentSearch.reset();
-                return false;
-        }
-
-        m_searchResults.clear();
-s
+m_currentS
-                boost::bind(&ShellCommands::helperOnSearchEvent, this, _1)
-        );
-
-        logMsg("Initialising new search.");
-s
+m_currentS
-
-        return true;
@@ -886 +892 @@
-        *m_socket << "cd            Change directory."          << Socket::Endl;
-        *m_socket << "uname         Show Hydranode version."    << Socket::Endl;
-        //*m_socket << "history       List command history"     << Socket::Endl;
-        *m_socket << "search        Search for files."          << Socket::Endl;
+        *m_socket << "ss            Stop current search."       << Socket::Endl;
-        *m_socket << "vr [pred]     View search results."       << Socket::Endl;
-        *m_socket << "download      Download a search result or link.";
@@ -899 +905 @@
-        *m_socket << "addsource     Add a source to download."  << Socket::Endl;
-        *m_socket << "hs            View Hasher Statistics."    << Socket::Endl;
-s 
+cs
-        *m_socket << "ifs           View IPFilter Statistics."  << Socket::Endl;
-        *m_socket << "log [on/off]  Enable/Disable log printing."<<Socket::Endl;
@@ -1410 +1416 @@
-        }
-        sortResults(pred);
-
+true
-        return true;
-}
@@ -1463 +1469 @@
-        fmt % Utils::bytesToString(avgUp) % Utils::bytesToString(avgDown);
-        *m_socket << fmt.str() << Socket::Endl;
+        return true;
+}
+
+bool ShellCommands::cmdStopSearch(Tokenizer) {
+        if (m_currentSearch) {
+                m_currentSearch->stop();
+                m_currentSearch.reset();
+        } else {
+                *m_socket << "'ss': Stops current active search.";
+                *m_socket << Socket::Endl;
+        }
-        return true;
-}
@@ -1624 +1641 @@
-
-void ShellCommands::helperOnSearchEvent(SearchPtr search) try {
-        CHECK_THROW(search->getResultCount() >= m_searchResults.size());
-
-        if (!search->getResultCount()) {
-                *m_socket << "\rReceived 0 results from server." <<Socket::Endl;
-                m_parent->setPrompt();
-                return;
-        }
@@ -1637 +1650 @@
-        }
-
-
-
-
-
-
+
+
-
-        m_parent->setPrompt();
@@ -1650 +1660 @@
-
-
-
+bool printAll
-        CHECK_THROW(m_socket);
-        if (!m_searchResults.size()) {
@@ -1656 +1666 @@
-        }
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-                SearchResultPtr sr(m_searchResults[i]);
-
-                fmt % i % Utils::bytesToString(sr->getSize());
-                fmt % sr->getSources() % sr->getComplete();
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-                *m_socket << fmt.str() << Socket::Endl;
+        }
+        if (toPrint < m_searchResults.size()) {
+                *m_socket << 
+                "  => Notice: Only the most popular results are printed. ";
+                *m_socket << Socket::Endl;
+                *m_socket << "  => To view all results, type 'vr'.";
+                *m_socket << " To stop searching, type 'ss'.";
+                *m_socket << Socket::Endl;
-        }
-}

hydranode/hncore/hnsh/shellcommands.h

@@ -198 +198 @@
-        bool cmdMemStats(Tokenizer);
-
+        //! Stops current search
+        bool cmdStopSearch(Tokenizer);
+
-        //! Print one client
-    &