Fix CVE-2017-11467: upgrade OrientDB 2.1.25 → 2.2.37

.github/workflows/build.yml

@@ -53,7 +53,7 @@ jobs:
           fi
           unzip openidm-zip/target/openidm-*.zip
           openidm/startup.sh &
-          timeout 3m bash -c 'until grep -q "OpenIDM ready" openidm/logs/openidm0.log.0 ; do sleep 5; done' || cat openidm/logs/openidm0.log.0
+          timeout 5m bash -c 'until grep -q "OpenIDM ready" openidm/logs/openidm0.log.0 ; do sleep 5; done' || cat openidm/logs/openidm0.log.0
           grep -q "OpenIDM ready" openidm/logs/openidm0.log.0
           ! grep "ERROR"  openidm/logs/openidm0.log.0
           ! grep "SEVERE" openidm/logs/openidm0.log.0
@@ -74,7 +74,7 @@ jobs:
           timeout 1m bash -c 'while [ -f openidm/.openidm.pid ]; do sleep 2; done' || true
           rm -rf openidm/logs/*
           OPENIDM_OPTS="-Dlogback.configurationFile=conf/logging-config.groovy -Dopenidm.context.path=/myidm" openidm/startup.sh &
-          timeout 3m bash -c 'until grep -q "OpenIDM ready" openidm/logs/openidm0.log.0 ; do sleep 5; done' || cat openidm/logs/openidm0.log.0
+          timeout 5m bash -c 'until grep -q "OpenIDM ready" openidm/logs/openidm0.log.0 ; do sleep 5; done' || cat openidm/logs/openidm0.log.0
           grep -q "OpenIDM ready" openidm/logs/openidm0.log.0
           ! grep "ERROR"  openidm/logs/openidm0.log.0
           ! grep "SEVERE" openidm/logs/openidm0.log.0

.gitignore

@@ -99,4 +99,7 @@ node_modules
 
 # ctags index
 tags
-test-output
+test-output
+
+# OrientDB test database files generated during tests
+openidm-repo-orientdb/databases/

legal/THIRDPARTYREADME.txt

@@ -226,23 +226,11 @@ Version: org.springframework:spring-beans:jar:2.5.6.SEC01
          org.springframework:spring-context:jar:2.5.6.SEC01
 Copyright:  Copyright 2002-2015 the original author or authors.
 
-Version: orient-commons-1.7.10.jar
-Copyright: Copyright 2010-2012 Luca Garulli (l.garulli(at)orientechnologies.com)
+Version: orientdb-core-2.2.37.jar
+Copyright: Copyright 2010-2018 Luca Garulli (l.garulli(at)orientechnologies.com)
 
-Version: orientdb-client-1.7.10.jar
-Copyright: Copyright 2010-2012 Luca Garulli (l.garulli(at)orientechnologies.com)
-
-Version: orientdb-core-1.7.10.jar
-Copyright: Copyright 2010-2012 Luca Garulli (l.garulli(at)orientechnologies.com)
-
-Version: orientdb-enterprise-1.7.10.jar
-Copyright: Copyright 2010-2012 Luca Garulli (l.garulli(at)orientechnologies.com)
-
-Version: orientdb-server-1.7.10.jar
-Copyright: Copyright 2010-2012 Luca Garulli (l.garulli(at)orientechnologies.com)
-
-Version: orientdb-nativeos-1.7.10.jar
-Copyright: Copyright 2010-2012 Luca Garulli (l.garulli(at)orientechnologies.com)
+Version: orientdb-server-2.2.37.jar
+Copyright: Copyright 2010-2018 Luca Garulli (l.garulli(at)orientechnologies.com)
 
 Version: pax-swissbox-extender-1.8.0.jar
 Copyright: Copyright 2007-2008 Alin Dreghiciu.

openidm-doc/src/main/asciidoc/install-guide/chap-install.adoc

@@ -558,14 +558,14 @@ When OpenIDM is running on the localhost system, you can access these UIs at `\h
 
 OpenIDM comes with an internal noSQL database, OrientDB, for use as the internal repository out of the box. This makes it easy to get started with OpenIDM. OrientDB is not supported for production use, however, so use a supported JDBC database when moving to production.
 
-To query the internal noSQL database, download and extract link:https://search.maven.org/remotecontent?filepath=com/orientechnologies/orientdb-community/1.7.10/orientdb-community-1.7.10-distribution.zip[OrientDB (version 1.7.10), window=\_blank]. You will find the shell console in the `bin` directory. Start the OrientDB console, using `console.sh` or `console.bat`, and connect to the running OpenIDM instance, with the `connect` command:
+To query the internal noSQL database, download and extract link:https://search.maven.org/remotecontent?filepath=com/orientechnologies/orientdb-community/2.2.37/orientdb-community-2.2.37.zip[OrientDB (version 2.2.37), window=\_blank]. You will find the shell console in the `bin` directory. Start the OrientDB console, using `console.sh` or `console.bat`, and connect to the running OpenIDM instance, with the `connect` command:
 
 [source, console]
 ----
-$ cd /path/to/orientdb-community-1.7.10/bin
+$ cd /path/to/orientdb-community-2.2.37/bin
 $ ./console.sh
 
-OrientDB console v.1.7.10 (build @BUILD@) www.orientechnologies.com
+OrientDB console v.2.2.37 (build @BUILD@) www.orientechnologies.com
 Type 'help' to display all the commands supported.
 
 Installing extensions for GREMLIN language v.2.5.0

openidm-repo-orientdb/pom.xml

@@ -99,11 +99,6 @@
                 </exclusion>
             </exclusions>
         </dependency>
-        <dependency>
-            <groupId>com.orientechnologies</groupId>
-            <artifactId>orientdb-enterprise</artifactId>
-            <version>${orientdb.version}</version>
-        </dependency>
         <dependency>
             <groupId>net.java.dev.jna</groupId>
             <artifactId>jna</artifactId>

openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/Activator.java

@@ -59,6 +59,21 @@ public void start(BundleContext context) {
             bootConfig.put(OrientDBRepoService.CONFIG_PASSWORD, repoConfig.get(OrientDBRepoService.CONFIG_PASSWORD.toLowerCase()).getObject());
             bootConfig.put(OrientDBRepoService.CONFIG_POOL_MIN_SIZE, repoConfig.get(OrientDBRepoService.CONFIG_POOL_MIN_SIZE.toLowerCase()).getObject());
             bootConfig.put(OrientDBRepoService.CONFIG_POOL_MAX_SIZE, repoConfig.get(OrientDBRepoService.CONFIG_POOL_MAX_SIZE.toLowerCase()).getObject());
+            // Forward the full DB schema (dbStructure) to the bootstrap repo so that
+            // all OrientDB classes/clusters/indexes are created during the first
+            // storage open.  DBHelper.checkDB temporarily disables USE_WAL while
+            // setting up the schema, but USE_WAL is read at storage-open time, so
+            // the disable only takes effect on the very first open.  If the
+            // bootstrap creates the storage with only the "config" class, the WAL
+            // writer is started and any later schema additions performed by the
+            // full-service activation pay the full O(N²) WAL flush cost.  By
+            // creating the entire schema during bootstrap (when USE_WAL is
+            // genuinely off) the second getPool call simply finds every class
+            // already present and skips all expensive schema work.
+            Object dbStructure = repoConfig.get(OrientDBRepoService.CONFIG_DB_STRUCTURE.toLowerCase()).getObject();
+            if (dbStructure != null) {
+                bootConfig.put(OrientDBRepoService.CONFIG_DB_STRUCTURE, dbStructure);
+            }
 
             // Init the bootstrap repo
             bootSvc = OrientDBRepoService.getRepoBootService(bootConfig);

openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/DBHelper.java

@@ -306,31 +306,110 @@ private static void warmUpPool(ODatabaseDocumentPool pool, String dbURL, String
     private static ODatabaseDocumentTx checkDB(String dbURL, String user, String password, JsonValue completeConfig)
             throws InvalidException {
 
+        // Disable WAL and per-operation fsyncs before opening or creating the storage.
+        // USE_WAL is read at storage-open time, so it must be changed here, before
+        // ODatabaseDocumentTx is constructed.  With WAL enabled (the default) OrientDB
+        // calls fdatasync() from a background thread every WAL_COMMIT_TIMEOUT ms, and
+        // the WAL grows with every new cluster or index.  On Linux/macOS this produces
+        // O(N²) I/O: creating the N-th class triggers a flush of the entire WAL that
+        // already contains records for the N-1 previous classes.  With 30 schema classes
+        // the total wall-clock time exceeds five minutes.
+        //
+        // The dominant cost on Linux is actually
+        // STORAGE_MAKE_FULL_CHECKPOINT_AFTER_CLUSTER_CREATE (default true): OrientDB
+        // performs a full storage checkpoint — fsync()ing every cluster file and the
+        // WAL — after EACH addCluster() call.  As clusters accumulate, each
+        // checkpoint takes longer (it scans/syncs all previously-created clusters),
+        // giving the classic O(N²) total time observed during schema setup.
+        //
+        // Disabling these flags for the schema setup is safe in both the new-DB and
+        // the existing-DB paths: getPool() is synchronized static, so only one thread
+        // at a time touches these process-global flags, and the single setup
+        // connection is the only live connection to this storage (the pool for this
+        // dbURL is not created until after checkDB() returns).  The storage is closed
+        // cleanly afterwards — all dirty pages are flushed to disk by the disk cache —
+        // so the pool that re-opens it next gets a consistent store with the original
+        // settings restored.
+        boolean origSyncOnUpdate = OGlobalConfiguration.STORAGE_CONFIGURATION_SYNC_ON_UPDATE.getValueAsBoolean();
+        boolean origWalSyncOnPageFlush = OGlobalConfiguration.WAL_SYNC_ON_PAGE_FLUSH.getValueAsBoolean();
+        boolean origUseWal = OGlobalConfiguration.USE_WAL.getValueAsBoolean();
+        boolean origCheckpointAfterClusterCreate =
+                OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_CLUSTER_CREATE.getValueAsBoolean();
+        boolean origCheckpointAfterCreate =
+                OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_CREATE.getValueAsBoolean();
+        boolean origCheckpointAfterOpen =
+                OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_OPEN.getValueAsBoolean();
+
         // TODO: Creation/opening of db may be not be necessary if we require this managed externally
-        ODatabaseDocumentTx db = new ODatabaseDocumentTx(dbURL);
+        ODatabaseDocumentTx db;
 
         // To add support for remote DB checking/creation one
         // would need to use OServerAdmin instead
         // boolean dbExists = new OServerAdmin(dbURL).connect(user, password).existsDatabase();
 
         // Local DB we can auto populate
         if (isLocalDB(dbURL) || isMemoryDB(dbURL)) {
-            if (db.exists()) {
-                logger.info("Using DB at {}", dbURL);
-                db.open(user, password);
-                populateSample(db, completeConfig);
-            } else {
-                logger.info("DB does not exist, creating {}", dbURL);
-                db.create();
-                // Delete default admin user
-                OSecurity security = db.getMetadata().getSecurity();
-                security.dropUser(OUser.ADMIN);
-                // Create new admin user with new username and password
-                security.createUser(user, password, security.getRole(ORole.ADMIN));
+            // Suppress all sync-on-write overhead for the duration of schema setup.
+            // These flags MUST be changed BEFORE constructing ODatabaseDocumentTx:
+            // OLocalPaginatedStorage reads OGlobalConfiguration.USE_WAL in its
+            // constructor (which is invoked from `new ODatabaseDocumentTx(url)`)
+            // and caches the value for the lifetime of the storage.  Setting
+            // USE_WAL=false after the storage is already constructed has no
+            // effect, leaving the WAL writer active and producing the O(N²)
+            // schema-creation slowdown observed on Linux/macOS CI runners.
+            //
+            // getPool() is synchronized static, so only one thread runs this
+            // code at a time; the temporary global-config changes cannot
+            // affect concurrent database operations.  The single setup
+            // connection is the only live connection to this storage (the
+            // pool for this dbURL is not created until after checkDB()
+            // returns), so it is safe to disable WAL globally here.
+            OGlobalConfiguration.STORAGE_CONFIGURATION_SYNC_ON_UPDATE.setValue(false);
+            OGlobalConfiguration.WAL_SYNC_ON_PAGE_FLUSH.setValue(false);
+            OGlobalConfiguration.USE_WAL.setValue(false);
+            // The real O(N²) culprit on Linux: by default OrientDB performs a full
+            // storage checkpoint (fsync of every cluster file plus WAL flush) after
+            // EACH addCluster() invocation.  Each checkpoint scans/syncs every
+            // previously-created cluster, so total cost is O(N²).  Disable for the
+            // duration of schema setup; we'll do a single explicit flush by closing
+            // the storage cleanly when populateSample returns.
+            OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_CLUSTER_CREATE.setValue(false);
+            OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_CREATE.setValue(false);
+            OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_OPEN.setValue(false);
+            try {
+                db = new ODatabaseDocumentTx(dbURL);
+                // exists() does not open the storage; it only probes the
+                // on-disk database files.  The storage engine has already
+                // been instantiated above with USE_WAL=false captured.
+                boolean exists = db.exists();
+
+                if (exists) {
+                    logger.info("Using DB at {}", dbURL);
+                    db.open(user, password);
+                } else {
+                    logger.info("DB does not exist, creating {}", dbURL);
+                    db.create();
+                    // Delete default admin user
+                    OSecurity security = db.getMetadata().getSecurity();
+                    security.dropUser(OUser.ADMIN);
+                    // Create new admin user with new username and password
+                    security.createUser(user, password, security.getRole(ORole.ADMIN));
+                }
                 populateSample(db, completeConfig);
+            } finally {
+                OGlobalConfiguration.STORAGE_CONFIGURATION_SYNC_ON_UPDATE.setValue(origSyncOnUpdate);
+                OGlobalConfiguration.WAL_SYNC_ON_PAGE_FLUSH.setValue(origWalSyncOnPageFlush);
+                OGlobalConfiguration.USE_WAL.setValue(origUseWal);
+                OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_CLUSTER_CREATE
+                        .setValue(origCheckpointAfterClusterCreate);
+                OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_CREATE
+                        .setValue(origCheckpointAfterCreate);
+                OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_OPEN
+                        .setValue(origCheckpointAfterOpen);
             }
         } else {
             logger.info("Using remote DB at {}", dbURL);
+            db = new ODatabaseDocumentTx(dbURL);
         }
         return db;
     }

openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/EmbeddedOServerService.java

@@ -241,7 +241,7 @@ protected OServerConfiguration getOrientDBConfig(JsonValue config) {
         if (dbURL == null || dbURL.isEmpty()) {
             dbURL = "plocal:" + IdentityServer.getFileForWorkingPath("db/openidm").getAbsolutePath();
         }
-        File dbFolder = IdentityServer.getFileForWorkingPath(dbURL.split(":")[1]);
+        File dbFolder = IdentityServer.getFileForWorkingPath(dbURL.substring(dbURL.indexOf(':') + 1));
 
         OServerStorageConfiguration storage2 = new OServerStorageConfiguration();
         storage2.name = "openidm";
@@ -269,7 +269,8 @@ protected OServerConfiguration getOrientDBConfig(JsonValue config) {
         };
         // OrientDB currently logs a warning if this is not set, 
         // although it should be taking the setting from the config above instead.
-        System.setProperty("ORIENTDB_HOME", dbFolder.getAbsolutePath());
+        // Point to the OpenIDM working directory so that config/security.json can be found there.
+        System.setProperty("ORIENTDB_HOME", IdentityServer.getFileForWorkingPath("").getAbsolutePath());
 
         return configuration;
     }

openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/OrientDBRepoService.java

@@ -88,7 +88,6 @@
 import com.orientechnologies.orient.core.index.OIndexException;
 import com.orientechnologies.orient.core.record.impl.ODocument;
 import com.orientechnologies.orient.core.storage.ORecordDuplicatedException;
-import com.orientechnologies.orient.core.version.OSimpleVersion;
 
 /**
  * Repository service implementation using OrientDB
@@ -418,7 +417,7 @@ public ResourceResponse delete(DeleteRequest request) throws ResourceException {
                 throw new NotFoundException("Object does not exist for delete on: " + request.getResourcePath());
             }
 
-            db.delete(existingDoc.getIdentity(), new OSimpleVersion(ver));
+            db.delete(existingDoc.getIdentity(), ver);
             logger.debug("delete for id succeeded: {} revision: {}", localId, request.getRevision());
             return DocumentUtil.toResource(existingDoc);
         } catch (ODatabaseException ex) {

openidm-zip/pom.xml

@@ -557,6 +557,11 @@
             <artifactId>javax.annotation-api</artifactId>
             <version>1.3.2</version>
         </dependency>
+        <dependency>
+            <groupId>com.google.code.findbugs</groupId>
+            <artifactId>jsr305</artifactId>
+            <version>3.0.2</version>
+        </dependency>
 
         <!-- jdbc drivers -->
 <!--         <dependency> -->
@@ -770,10 +775,9 @@
                                     <artifactId>orientdb-community</artifactId>
                                     <version>${orientdb.studio.version}</version>
                                     <type>zip</type>
-                                    <classifier>distribution</classifier>
                                     <overWrite>false</overWrite>
                                     <outputDirectory>${project.build.directory}/</outputDirectory>
-                                    <includes>**/www/**,**/plugins/studio-*.zip</includes>
+                                    <includes>**/www/**,**/plugins/*studio*.zip</includes>
                                 </artifactItem>
                             </artifactItems>
                             <overWriteReleases>false</overWriteReleases>
@@ -792,8 +796,8 @@
                         <configuration>
                             <target>
                                 <unzip dest="${project.build.directory}/orientdb-community-${orientdb.studio.version}">
-                                    <fileset dir="${project.build.directory}/orientdb-community-${orientdb.studio.version}\plugins">
-                                        <include name="studio-**.zip" />
+                                    <fileset dir="${project.build.directory}/orientdb-community-${orientdb.studio.version}/plugins">
+                                        <include name="*studio*.zip" />
                                     </fileset>
                                 </unzip>
                             </target>

openidm-zip/src/main/resources/config/security.json

@@ -0,0 +1,6 @@
+{
+  "security": {
+    "enabled": false,
+    "debug": false
+  }
+}

openidm-zip/src/main/resources/startup.bat

@@ -35,7 +35,7 @@ for /f "delims=.-_ tokens=1-2" %%v in ("%JAVA_VERSION%") do (
 )
 set "ADD_OPENS_ARGS="
 if /I %JAVA_VERSION% GEQ 9 (
-  set ADD_OPENS_ARGS=--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED
+  set ADD_OPENS_ARGS=--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED
 )
 
 set "JPDA="

openidm-zip/src/main/resources/startup.sh

@@ -43,6 +43,7 @@ if [ $JAVA_VER -ge 90 ]; then
         --add-opens=java.base/java.lang=ALL-UNNAMED
         --add-opens=java.base/java.net=ALL-UNNAMED
         --add-opens=java.base/java.util=ALL-UNNAMED
+        --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
         --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED"
 fi