[Debtags-commits] [svn] r2103 - in daemon: . src
Enrico Zini
enrico at alioth.debian.org
Sun Nov 19 16:36:21 CET 2006
Author: enrico
Date: Sun Nov 19 16:36:21 2006
New Revision: 2103
Modified:
daemon/ (props changed)
daemon/src/cmdline.h
daemon/src/config.cpp
daemon/src/config.h
daemon/src/debtagsd.cpp
Log:
r3703 at viaza: enrico | 2006-11-19 16:35:41 +0100
Added popcon support and UNT function to get untagged items sorted by popcon
Modified: daemon/src/cmdline.h
==============================================================================
--- daemon/src/cmdline.h (original)
+++ daemon/src/cmdline.h Sun Nov 19 16:36:21 2006
@@ -43,6 +43,7 @@
StringOption* tagdb;
StringOption* vocabulary;
StringOption* pkgdb;
+ StringOption* popcondb;
StringOption* logfile;
StringOption* patchdir;
@@ -81,6 +82,8 @@
"pathname of the plaintext vocabulary file");
pkgdb = startOpts->add<StringOption>("pkgdb", 0, "pkgdb", "pathname",
"pathname of the file with package descriptions");
+ popcondb = startOpts->add<StringOption>("popcondb", 0, "popcondb", "pathname",
+ "pathname of the file with popcon vote rankings");
logfile = startOpts->add<StringOption>("logfile", 0, "logfile", "pathname",
"pathname of the log file where to write what happens with debtagsd");
patchdir = startOpts->add<StringOption>("patchdir", 0, "patchdir", "pathname",
Modified: daemon/src/config.cpp
==============================================================================
--- daemon/src/config.cpp (original)
+++ daemon/src/config.cpp Sun Nov 19 16:36:21 2006
@@ -54,6 +54,8 @@
vocabulary = opts.vocabulary->stringValue();
if (opts.pkgdb->boolValue())
pkgdb = opts.pkgdb->stringValue();
+ if (opts.popcondb->boolValue())
+ popcondb = opts.popcondb->stringValue();
if (opts.logfile->boolValue())
logfile = opts.logfile->stringValue();
if (opts.patchdir->boolValue())
Modified: daemon/src/config.h
==============================================================================
--- daemon/src/config.h (original)
+++ daemon/src/config.h Sun Nov 19 16:36:21 2006
@@ -18,6 +18,7 @@
std::string tagdb;
std::string vocabulary;
std::string pkgdb;
+ std::string popcondb;
std::string logfile;
std::string patchdir;
int kill_signal;
@@ -33,6 +34,7 @@
tagdb("/var/lib/debtags/package-tags"),
vocabulary("/var/lib/debtags/vocabulary"),
pkgdb("/var/lib/dpkg/available"),
+ popcondb("/var/lib/debtags/popcon"),
logfile("/var/log/debtagsd.log"),
patchdir("/var/lib/debtags/patches"),
kill_signal(15),
Modified: daemon/src/debtagsd.cpp
==============================================================================
--- daemon/src/debtagsd.cpp (original)
+++ daemon/src/debtagsd.cpp Sun Nov 19 16:36:21 2006
@@ -109,153 +109,218 @@
Connection(int fd) : fd(fd), m_eof(false) {}
~Connection() { close(fd); }
- bool eof() const { return m_eof; }
+bool eof() const { return m_eof; }
- // Read a line
- std::string readline()
+// Read a line
+std::string readline()
+{
+ // TODO: use a timeout
+ std::string line;
+ char c;
+ while (true)
{
- // TODO: use a timeout
- std::string line;
- char c;
- while (true)
+ int res = read(fd, &c, 1);
+ if (res == 0)
{
- int res = read(fd, &c, 1);
- if (res == 0)
- {
- m_eof = true;
- break;
- }
- if (res < 0)
- throw wibble::exception::System("reading data");
- if (c == '\n')
- break;
- line += c;
+ m_eof = true;
+ break;
}
- return line;
+ if (res < 0)
+ throw wibble::exception::System("reading data");
+ if (c == '\n')
+ break;
+ line += c;
}
+ return line;
+}
- // Consume all the input
- void discardRemainingInput()
+// Consume all the input
+void discardRemainingInput()
+{
+ char buf[1024];
+ while (true)
{
- char buf[1024];
- while (true)
+ int res = read(fd, buf, 1024);
+ if (res == 0)
{
- int res = read(fd, buf, 1024);
- if (res == 0)
- {
- m_eof = true;
- break;
- }
- if (res < 0)
- throw wibble::exception::System("reading data");
+ m_eof = true;
+ break;
}
+ if (res < 0)
+ throw wibble::exception::System("reading data");
}
+}
- // Write a string
- void write(const std::string& str)
- {
- size_t res = ::write(fd, str.data(), str.size());
- if (res != str.size())
- throw wibble::exception::System("writing data");
- }
-
- void credentials(uid_t& uid, gid_t& gid)
- {
- struct ucred cr;
- socklen_t cl = sizeof(cr);
- if (getsockopt(fd, SOL_SOCKET, SO_PEERCRED, &cr, &cl) == -1)
- throw wibble::exception::System("reading socket peer credentials");
- uid = cr.uid;
- gid = cr.gid;
- }
+// Write a string
+void write(const std::string& str)
+{
+ size_t res = ::write(fd, str.data(), str.size());
+ if (res != str.size())
+ throw wibble::exception::System("writing data");
+}
+
+void credentials(uid_t& uid, gid_t& gid)
+{
+ struct ucred cr;
+ socklen_t cl = sizeof(cr);
+ if (getsockopt(fd, SOL_SOCKET, SO_PEERCRED, &cr, &cl) == -1)
+ throw wibble::exception::System("reading socket peer credentials");
+ uid = cr.uid;
+ gid = cr.gid;
+}
};
/// Deserialize a sequence of keywords
std::vector<std::string> parseKeys(const std::string& str)
{
- std::vector<std::string> res;
- wibble::Tokenizer tok(str, "[[:alnum:]_-]+", REG_EXTENDED);
- for (wibble::Tokenizer::const_iterator i = tok.begin(); i != tok.end(); ++i)
- res.push_back(*i);
- return res;
+std::vector<std::string> res;
+wibble::Tokenizer tok(str, "[[:alnum:]_-]+", REG_EXTENDED);
+for (wibble::Tokenizer::const_iterator i = tok.begin(); i != tok.end(); ++i)
+ res.push_back(*i);
+return res;
}
/// Deserialize a sequence of tags
std::set<std::string> parseTagset(const std::string& str)
{
- std::set<std::string> res;
- wibble::Tokenizer tok(str, "[[:alnum:]:_-]+", REG_EXTENDED);
- for (wibble::Tokenizer::const_iterator i = tok.begin(); i != tok.end(); ++i)
- res.insert(*i);
- return res;
+std::set<std::string> res;
+wibble::Tokenizer tok(str, "[[:alnum:]:_-]+", REG_EXTENDED);
+for (wibble::Tokenizer::const_iterator i = tok.begin(); i != tok.end(); ++i)
+ res.insert(*i);
+return res;
}
class Tags : public std::set<std::string>
{
public:
- Tags(Config& cfg)
+Tags(Config& cfg)
+{
+ tagcoll::input::Stdio in(cfg.vocabulary);
+ DebDBParser parser(in);
+ DebDBParser::Record rec;
+ while (parser.nextRecord(rec))
{
- tagcoll::input::Stdio in(cfg.vocabulary);
- DebDBParser parser(in);
- DebDBParser::Record rec;
- while (parser.nextRecord(rec))
- {
- DebDBParser::Record::const_iterator i = rec.find("Tag");
- if (i != rec.end()) insert(i->second);
- }
+ DebDBParser::Record::const_iterator i = rec.find("Tag");
+ if (i != rec.end()) insert(i->second);
}
+}
- bool has(const std::string& tag)
- {
- return find(tag) != end();
- }
+bool has(const std::string& tag)
+{
+ return find(tag) != end();
+}
};
/// Quick access to the short descriptions of every package
class Descs : public std::map<std::string, std::pair<std::string, std::string> >
{
- std::pair<std::string, std::string> none;
+std::pair<std::string, std::string> none;
+public:
+Descs(Config& cfg)
+{
+ tagcoll::input::Stdio in(cfg.pkgdb);
+ DebDBParser parser(in);
+ DebDBParser::Record rec;
+ while (parser.nextRecord(rec))
+ {
+ size_t sd = rec["Description"].find("\n");
+ if (sd == string::npos)
+ insert(make_pair(rec["Package"],
+ make_pair(rec["Description"], string())));
+ else
+ insert(make_pair(rec["Package"],
+ make_pair(
+ rec["Description"].substr(0, sd),
+ rec["Description"].substr(sd+1))));
+ }
+}
+
+bool has(const std::string& pkg) const
+{
+ return find(pkg) != end();
+}
+
+const std::pair<std::string, std::string>& get(const std::string& pkg) const
+{
+ const_iterator i = find(pkg);
+ if (i == end())
+ return none;
+ else
+ return i->second;
+}
+
+std::set<std::string> allNames() const
+{
+ std::set<std::string> res;
+ for (const_iterator i = begin(); i != end(); ++i)
+ res.insert(i->first);
+ return res;
+}
+};
+
+// Quick access to popcon votes
+class Popcon : public std::map<std::string, int>
+{
+public:
+class RankSort
+{
+ const Popcon& p;
public:
- Descs(Config& cfg)
+ RankSort(const Popcon& p) : p(p) {}
+ bool operator()(const std::string& t1, const std::string& t2)
{
- tagcoll::input::Stdio in(cfg.pkgdb);
- DebDBParser parser(in);
- DebDBParser::Record rec;
- while (parser.nextRecord(rec))
+ return p.get(t1) > p.get(t2);
+ }
+};
+
+RankSort rankSort() const { return RankSort(*this); }
+
+std::vector<std::string> ranked;
+
+Popcon(Config& cfg)
+{
+ std::ifstream in(cfg.popcondb.c_str());
+ string line;
+ while (getline(in, line))
+ {
+ // Skip comments
+ if (line[0] == '#') continue;
+ // Terminate before the totals
+ if (line[0] == '-') break;
+
+ // Read package name
+ size_t i = 6;
+ for ( ; i < line.size() && !isspace(line[i]); ++i)
+ ;
+ string name = line.substr(6, i-6);
+
+ // Read vote count
+ for (i = 43; i < line.size() && isspace(line[i]); ++i)
+ ;
+ int vote = -1;
+ if (i < line.size() && isdigit(line[i]))
+ vote = strtoul(line.substr(i).c_str(), NULL, 10);
+
+ // Acquire the data if valid
+ if (!name.empty() && vote != -1)
{
- size_t sd = rec["Description"].find("\n");
- if (sd == string::npos)
- insert(make_pair(rec["Package"],
- make_pair(rec["Description"], string())));
- else
- insert(make_pair(rec["Package"],
- make_pair(
- rec["Description"].substr(0, sd),
- rec["Description"].substr(sd+1))));
+ insert(make_pair(name, vote));
+ ranked.push_back(name);
}
}
-
- bool has(const std::string& pkg) const
- {
- return find(pkg) != end();
+ // Sort the ranked vector by popcon votes
+ // (not needed because the file is sorted already)
+ //std::sort(ranked.begin(), ranked.end(), rankSort());
}
- const std::pair<std::string, std::string>& get(const std::string& pkg) const
+ int get(const std::string& pkg) const
{
const_iterator i = find(pkg);
if (i == end())
- return none;
+ return 0;
else
return i->second;
}
-
- std::set<std::string> allNames() const
- {
- std::set<std::string> res;
- for (const_iterator i = begin(); i != end(); ++i)
- res.insert(i->first);
- return res;
- }
};
/// The debtagsd daemon process
@@ -266,6 +331,7 @@
tagcoll::coll::Fast<std::string, std::string> coll;
Descs desc;
+ Popcon popcon;
Tags tags;
FullTextSearch fts;
@@ -336,6 +402,27 @@
i != pkgs.end(); ++i)
outputPackage(*i, conn);
}
+ else if (cmd == "UNT")
+ {
+ // Read the maximum number of tags allowed per package (default to 0)
+ size_t count = 0;
+ if (!line.empty() && isdigit(line[0]))
+ count = strtoul(line.c_str(), NULL, 10);
+
+ // Send the top 500 untagged items, by popcon ordering
+ int sent = 0;
+ for (vector<string>::const_iterator i = popcon.ranked.begin();
+ i != popcon.ranked.end(); ++i)
+ {
+ // Only consider the packages that are in our package database
+ if (!desc.has(*i)) continue;
+ if (coll.getTagsOfItem(*i).size() <= count)
+ {
+ if (++sent > 500) break;
+ outputPackage(*i, conn);
+ }
+ }
+ }
else if (cmd == "FTS")
{
vector<string> terms = parseKeys(line);
@@ -444,6 +531,7 @@
{
try {
// Read the submitter tag
+ // (see http://lists.alioth.debian.org/pipermail/debtags-devel/2006-July/001290.html)
std::string submitter = line;
// Read the patch in `buf`
@@ -541,7 +629,7 @@
}
public:
- Debtagsd(Config& cfg) : cfg(cfg), desc(cfg), tags(cfg), fts(cfg), seq(0)
+ Debtagsd(Config& cfg) : cfg(cfg), desc(cfg), popcon(cfg), tags(cfg), fts(cfg), seq(0)
{
tagcoll::input::Stdio input(cfg.tagdb);
tagcoll::textformat::parse(input, tagcoll::coll::inserter(coll));
More information about the Debtags-commits
mailing list