GitHub Issue 1579: Don't log indexing errors about very long paths

search/src/org/labkey/search/model/DavCrawler.java

@@ -91,10 +91,6 @@ public class DavCrawler implements ShutdownListener
     // 1 Mbyte/sec, this seems to be enough to use a LOT of tika cpu time
     final RateLimiter _fileIORateLimiter = new RateLimiter("file io", 1000000, TimeUnit.SECONDS);
 
-    // CONSIDER: file count limiter
-    final RateLimiter _filesIndexRateLimiter = new RateLimiter("file index", 100, TimeUnit.SECONDS);
-
-
     public static class ResourceInfo
     {
         ResourceInfo(Date indexed, Date modified)
@@ -616,10 +612,10 @@ static boolean skipContainer(WebdavResource r)
         if (null != f)
         {
             // labkey convention
-            if (new File(f,".nocrawl").exists())
+            if (FileUtil.appendName(f,".nocrawl").exists())
                 return true;
             // postgres
-            if (new File(f,"PG_VERSION").exists())
+            if (FileUtil.appendName(f,"PG_VERSION").exists())
                 return true;
         }
 

search/src/org/labkey/search/model/SavePaths.java

@@ -144,7 +144,7 @@ private synchronized int getId(Path path) throws SQLException
     }
 
 
-    // create if not exists
+    // create if not exists; -1 if the parent can't be stored
     private synchronized int _getParentId(Path path) throws SQLException
     {
         Path parent = path.getParent();
@@ -164,14 +164,20 @@ private synchronized int _getParentId(Path path) throws SQLException
 
 
 
+    /** @return the collection's id, or -1 if it can't be stored */
     private int _ensure(Path path) throws SQLException
     {
+        if (!checkLengths(path))
+            return -1;
+
         // Mostly I don't care about Parent
         // However, we need this for the primary key
         int valueParent = _getParentId(path);
+        if (-1 == valueParent)
+            return -1;
 
         String valuePath = toPathString(path);
-        String valueName = path.equals(Path.rootPath) ? "/" : path.getName();   // "" is treated like NULL
+        String valueName = collectionName(path);
         Date valueNextCrawl = new Date();
 
         DbSchema db = getSearchSchema();
@@ -181,9 +187,10 @@ private int _ensure(Path path) throws SQLException
         try
         {
             SQLFragment insert = new SQLFragment(
-                    "INSERT INTO search.crawlcollections (parent, name, path, lastcrawled, nextcrawl)\n" +
-                    "SELECT ? as parent, ? as name, ? as path, ? as lastcrawled, ? as nextcrawl\n" +
-                    "WHERE NOT EXISTS (SELECT * FROM search.crawlcollections WHERE parent=? and name=?)");
+                    """
+                            INSERT INTO search.crawlcollections (parent, name, path, lastcrawled, nextcrawl)
+                            SELECT ? as parent, ? as name, ? as path, ? as lastcrawled, ? as nextcrawl
+                            WHERE NOT EXISTS (SELECT * FROM search.crawlcollections WHERE parent=? and name=?)""");
             // values
             insert.add(valueParent);
             insert.add(valueName);
@@ -218,14 +225,45 @@ private int _ensure(Path path) throws SQLException
     }
 
 
+    // "" is treated like NULL, so the root collection is stored as "/"
+    private static String collectionName(Path path)
+    {
+        return path.equals(Path.rootPath) ? "/" : path.getName();
+    }
+
+
+    // A value too long for its column can never be stored, so warn and skip instead of letting the INSERT throw on every crawl
+    private boolean checkLengths(Path path)
+    {
+        TableInfo coll = getSearchSchema().getTable("CrawlCollections");
+        return checkLength(coll.getColumn("Path"), toPathString(path), path) &&
+                checkLength(coll.getColumn("Name"), collectionName(path), path);
+    }
+
+
+    private boolean checkLength(ColumnInfo column, String value, Path path)
+    {
+        if (value.length() <= column.getScale())
+            return true;
+        _log.warn("Skipping '{}': {} value is {} characters, which exceeds the maximum of {}", path, column.getName(), value.length(), column.getScale());
+        return false;
+    }
+
+
     @Override
     public boolean insertPath(Path path, Date nextCrawl)
     {
+        if (!checkLengths(path))
+            return false;
+
         try
         {
             // Mostly I don't care about Parent
             // However, we need this for the primary key
             int parent = _getParentId(path);
+            if (-1 == parent)
+                return false;
+
             if (nextCrawl == null)
                 nextCrawl = new Date(System.currentTimeMillis()+5*60000);
 
@@ -235,7 +273,7 @@ public boolean insertPath(Path path, Date nextCrawl)
                     "SELECT ?,?,?,?,? " +
                     "WHERE NOT EXISTS (SELECT Path FROM search.crawlcollections WHERE ");
             f.add(pathStr);
-            f.add(path.equals(Path.rootPath) ? "/" : path.getName());   // "" is treated like NULL
+            f.add(collectionName(path));
             f.add(parent);
             f.add(nextCrawl);
             f.add(nullDate);
@@ -263,6 +301,9 @@ public boolean insertPath(Path path, Date nextCrawl)
     @Override
     public boolean updatePath(Path path, java.util.Date last, java.util.Date next, boolean create)
     {
+        if (!checkLengths(path))
+            return false;
+
         try
         {
             boolean success = _update(path,last,next);
@@ -343,13 +384,12 @@ public Map<Path, Pair<Date,Date>> getPaths(int limit)
         Date awhileago = new Date(Math.max(_startupTime, now.getTime() - 30*60000));
 
         SqlDialect dialect = getSearchSchema().getSqlDialect();
-        SQLFragment f = new SQLFragment(
-                "SELECT Parent, Name, Path, LastCrawled, NextCrawl\n" +
-                "FROM search.CrawlCollections\n");
-        f.append("WHERE NextCrawl < ? AND (LastCrawled IS NULL OR LastCrawled < ?) " +
-                "ORDER BY NextCrawl");
-        f.add(now);
-        f.add(awhileago);
+        SQLFragment f = new SQLFragment("""
+                SELECT Parent, Name, Path, LastCrawled, NextCrawl
+                FROM search.CrawlCollections
+                WHERE NextCrawl < ? AND (LastCrawled IS NULL OR LastCrawled < ?)
+                ORDER BY NextCrawl
+                """, now, awhileago);
         SQLFragment sel = dialect.limitRows(f, limit);
 
         try
@@ -407,9 +447,10 @@ public Map<Path, Pair<Date,Date>> getPaths(int limit)
     public Map<String, DavCrawler.ResourceInfo> getFiles(Path path)
     {
         SQLFragment s = new SQLFragment(
-                "SELECT D.ChangeInterval, D.Path, D.id, F.Name, F.Modified, F.LastIndexed\n" +
-                "FROM search.CrawlCollections D LEFT OUTER JOIN search.CrawlResources F on D.id=F.parent\n" +
-                "WHERE D.path = ?");
+                """
+                        SELECT D.ChangeInterval, D.Path, D.id, F.Name, F.Modified, F.LastIndexed
+                        FROM search.CrawlCollections D LEFT OUTER JOIN search.CrawlResources F on D.id=F.parent
+                        WHERE D.path = ?""");
         s.add(toPathString(path));
 
         final Map<String,DavCrawler.ResourceInfo> map = new HashMap<>();
@@ -431,13 +472,18 @@ public Map<String, DavCrawler.ResourceInfo> getFiles(Path path)
     @Override
     public boolean updateFile(@NotNull Path path, @NotNull Date lastIndexed, Date modified)
     {
+        if (!checkLength(getSearchSchema().getTable("CrawlResources").getColumn("Name"), path.getName(), path))
+            return false;
+
         try
         {
             if (null == datetime)
                 datetime = getSearchSchema().getSqlDialect().getDefaultDateTimeDataType();
             if (modified.getTime() == Long.MIN_VALUE)
                 modified = null;
             int id = _getParentId(path);
+            if (-1 == id)
+                return false;
             SQLFragment upd = new SQLFragment(
                     "UPDATE search.CrawlResources SET LastIndexed=?, Modified=CAST(? AS " + datetime + ") WHERE Parent=? AND Name=?",
                     lastIndexed, modified, id, path.getName());