diff --git a/search/src/org/labkey/search/model/DavCrawler.java b/search/src/org/labkey/search/model/DavCrawler.java index 9a01cc7052c..764c3ee45d2 100644 --- a/search/src/org/labkey/search/model/DavCrawler.java +++ b/search/src/org/labkey/search/model/DavCrawler.java @@ -91,10 +91,6 @@ public class DavCrawler implements ShutdownListener // 1 Mbyte/sec, this seems to be enough to use a LOT of tika cpu time final RateLimiter _fileIORateLimiter = new RateLimiter("file io", 1000000, TimeUnit.SECONDS); - // CONSIDER: file count limiter - final RateLimiter _filesIndexRateLimiter = new RateLimiter("file index", 100, TimeUnit.SECONDS); - - public static class ResourceInfo { ResourceInfo(Date indexed, Date modified) @@ -616,10 +612,10 @@ static boolean skipContainer(WebdavResource r) if (null != f) { // labkey convention - if (new File(f,".nocrawl").exists()) + if (FileUtil.appendName(f,".nocrawl").exists()) return true; // postgres - if (new File(f,"PG_VERSION").exists()) + if (FileUtil.appendName(f,"PG_VERSION").exists()) return true; } diff --git a/search/src/org/labkey/search/model/SavePaths.java b/search/src/org/labkey/search/model/SavePaths.java index 40ef93a7e8f..efa5cbfd6e0 100644 --- a/search/src/org/labkey/search/model/SavePaths.java +++ b/search/src/org/labkey/search/model/SavePaths.java @@ -144,7 +144,7 @@ private synchronized int getId(Path path) throws SQLException } - // create if not exists + // create if not exists; -1 if the parent can't be stored private synchronized int _getParentId(Path path) throws SQLException { Path parent = path.getParent(); @@ -164,14 +164,20 @@ private synchronized int _getParentId(Path path) throws SQLException + /** @return the collection's id, or -1 if it can't be stored */ private int _ensure(Path path) throws SQLException { + if (!checkLengths(path)) + return -1; + // Mostly I don't care about Parent // However, we need this for the primary key int valueParent = _getParentId(path); + if (-1 == valueParent) + return -1; String valuePath = toPathString(path); - String valueName = path.equals(Path.rootPath) ? "/" : path.getName(); // "" is treated like NULL + String valueName = collectionName(path); Date valueNextCrawl = new Date(); DbSchema db = getSearchSchema(); @@ -181,9 +187,10 @@ private int _ensure(Path path) throws SQLException try { SQLFragment insert = new SQLFragment( - "INSERT INTO search.crawlcollections (parent, name, path, lastcrawled, nextcrawl)\n" + - "SELECT ? as parent, ? as name, ? as path, ? as lastcrawled, ? as nextcrawl\n" + - "WHERE NOT EXISTS (SELECT * FROM search.crawlcollections WHERE parent=? and name=?)"); + """ + INSERT INTO search.crawlcollections (parent, name, path, lastcrawled, nextcrawl) + SELECT ? as parent, ? as name, ? as path, ? as lastcrawled, ? as nextcrawl + WHERE NOT EXISTS (SELECT * FROM search.crawlcollections WHERE parent=? and name=?)"""); // values insert.add(valueParent); insert.add(valueName); @@ -218,14 +225,45 @@ private int _ensure(Path path) throws SQLException } + // "" is treated like NULL, so the root collection is stored as "/" + private static String collectionName(Path path) + { + return path.equals(Path.rootPath) ? "/" : path.getName(); + } + + + // A value too long for its column can never be stored, so warn and skip instead of letting the INSERT throw on every crawl + private boolean checkLengths(Path path) + { + TableInfo coll = getSearchSchema().getTable("CrawlCollections"); + return checkLength(coll.getColumn("Path"), toPathString(path), path) && + checkLength(coll.getColumn("Name"), collectionName(path), path); + } + + + private boolean checkLength(ColumnInfo column, String value, Path path) + { + if (value.length() <= column.getScale()) + return true; + _log.warn("Skipping '{}': {} value is {} characters, which exceeds the maximum of {}", path, column.getName(), value.length(), column.getScale()); + return false; + } + + @Override public boolean insertPath(Path path, Date nextCrawl) { + if (!checkLengths(path)) + return false; + try { // Mostly I don't care about Parent // However, we need this for the primary key int parent = _getParentId(path); + if (-1 == parent) + return false; + if (nextCrawl == null) nextCrawl = new Date(System.currentTimeMillis()+5*60000); @@ -235,7 +273,7 @@ public boolean insertPath(Path path, Date nextCrawl) "SELECT ?,?,?,?,? " + "WHERE NOT EXISTS (SELECT Path FROM search.crawlcollections WHERE "); f.add(pathStr); - f.add(path.equals(Path.rootPath) ? "/" : path.getName()); // "" is treated like NULL + f.add(collectionName(path)); f.add(parent); f.add(nextCrawl); f.add(nullDate); @@ -263,6 +301,9 @@ public boolean insertPath(Path path, Date nextCrawl) @Override public boolean updatePath(Path path, java.util.Date last, java.util.Date next, boolean create) { + if (!checkLengths(path)) + return false; + try { boolean success = _update(path,last,next); @@ -343,13 +384,12 @@ public Map> getPaths(int limit) Date awhileago = new Date(Math.max(_startupTime, now.getTime() - 30*60000)); SqlDialect dialect = getSearchSchema().getSqlDialect(); - SQLFragment f = new SQLFragment( - "SELECT Parent, Name, Path, LastCrawled, NextCrawl\n" + - "FROM search.CrawlCollections\n"); - f.append("WHERE NextCrawl < ? AND (LastCrawled IS NULL OR LastCrawled < ?) " + - "ORDER BY NextCrawl"); - f.add(now); - f.add(awhileago); + SQLFragment f = new SQLFragment(""" + SELECT Parent, Name, Path, LastCrawled, NextCrawl + FROM search.CrawlCollections + WHERE NextCrawl < ? AND (LastCrawled IS NULL OR LastCrawled < ?) + ORDER BY NextCrawl + """, now, awhileago); SQLFragment sel = dialect.limitRows(f, limit); try @@ -407,9 +447,10 @@ public Map> getPaths(int limit) public Map getFiles(Path path) { SQLFragment s = new SQLFragment( - "SELECT D.ChangeInterval, D.Path, D.id, F.Name, F.Modified, F.LastIndexed\n" + - "FROM search.CrawlCollections D LEFT OUTER JOIN search.CrawlResources F on D.id=F.parent\n" + - "WHERE D.path = ?"); + """ + SELECT D.ChangeInterval, D.Path, D.id, F.Name, F.Modified, F.LastIndexed + FROM search.CrawlCollections D LEFT OUTER JOIN search.CrawlResources F on D.id=F.parent + WHERE D.path = ?"""); s.add(toPathString(path)); final Map map = new HashMap<>(); @@ -431,6 +472,9 @@ public Map getFiles(Path path) @Override public boolean updateFile(@NotNull Path path, @NotNull Date lastIndexed, Date modified) { + if (!checkLength(getSearchSchema().getTable("CrawlResources").getColumn("Name"), path.getName(), path)) + return false; + try { if (null == datetime) @@ -438,6 +482,8 @@ public boolean updateFile(@NotNull Path path, @NotNull Date lastIndexed, Date mo if (modified.getTime() == Long.MIN_VALUE) modified = null; int id = _getParentId(path); + if (-1 == id) + return false; SQLFragment upd = new SQLFragment( "UPDATE search.CrawlResources SET LastIndexed=?, Modified=CAST(? AS " + datetime + ") WHERE Parent=? AND Name=?", lastIndexed, modified, id, path.getName());