diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d7227ca04d..b305c1b4af 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,7 +53,7 @@ jobs: fi unzip openidm-zip/target/openidm-*.zip openidm/startup.sh & - timeout 3m bash -c 'until grep -q "OpenIDM ready" openidm/logs/openidm0.log.0 ; do sleep 5; done' || cat openidm/logs/openidm0.log.0 + timeout 5m bash -c 'until grep -q "OpenIDM ready" openidm/logs/openidm0.log.0 ; do sleep 5; done' || cat openidm/logs/openidm0.log.0 grep -q "OpenIDM ready" openidm/logs/openidm0.log.0 ! grep "ERROR" openidm/logs/openidm0.log.0 ! grep "SEVERE" openidm/logs/openidm0.log.0 @@ -74,7 +74,7 @@ jobs: timeout 1m bash -c 'while [ -f openidm/.openidm.pid ]; do sleep 2; done' || true rm -rf openidm/logs/* OPENIDM_OPTS="-Dlogback.configurationFile=conf/logging-config.groovy -Dopenidm.context.path=/myidm" openidm/startup.sh & - timeout 3m bash -c 'until grep -q "OpenIDM ready" openidm/logs/openidm0.log.0 ; do sleep 5; done' || cat openidm/logs/openidm0.log.0 + timeout 5m bash -c 'until grep -q "OpenIDM ready" openidm/logs/openidm0.log.0 ; do sleep 5; done' || cat openidm/logs/openidm0.log.0 grep -q "OpenIDM ready" openidm/logs/openidm0.log.0 ! grep "ERROR" openidm/logs/openidm0.log.0 ! grep "SEVERE" openidm/logs/openidm0.log.0 diff --git a/.gitignore b/.gitignore index ad3e5f7180..fbdd7b3ef9 100644 --- a/.gitignore +++ b/.gitignore @@ -99,4 +99,7 @@ node_modules # ctags index tags -test-output \ No newline at end of file +test-output + +# OrientDB test database files generated during tests +openidm-repo-orientdb/databases/ \ No newline at end of file diff --git a/legal/THIRDPARTYREADME.txt b/legal/THIRDPARTYREADME.txt index 0d97c222f9..1474fa6e6f 100644 --- a/legal/THIRDPARTYREADME.txt +++ b/legal/THIRDPARTYREADME.txt @@ -226,23 +226,11 @@ Version: org.springframework:spring-beans:jar:2.5.6.SEC01 org.springframework:spring-context:jar:2.5.6.SEC01 Copyright: Copyright 2002-2015 the original author or authors. -Version: orient-commons-1.7.10.jar -Copyright: Copyright 2010-2012 Luca Garulli (l.garulli(at)orientechnologies.com) +Version: orientdb-core-2.2.37.jar +Copyright: Copyright 2010-2018 Luca Garulli (l.garulli(at)orientechnologies.com) -Version: orientdb-client-1.7.10.jar -Copyright: Copyright 2010-2012 Luca Garulli (l.garulli(at)orientechnologies.com) - -Version: orientdb-core-1.7.10.jar -Copyright: Copyright 2010-2012 Luca Garulli (l.garulli(at)orientechnologies.com) - -Version: orientdb-enterprise-1.7.10.jar -Copyright: Copyright 2010-2012 Luca Garulli (l.garulli(at)orientechnologies.com) - -Version: orientdb-server-1.7.10.jar -Copyright: Copyright 2010-2012 Luca Garulli (l.garulli(at)orientechnologies.com) - -Version: orientdb-nativeos-1.7.10.jar -Copyright: Copyright 2010-2012 Luca Garulli (l.garulli(at)orientechnologies.com) +Version: orientdb-server-2.2.37.jar +Copyright: Copyright 2010-2018 Luca Garulli (l.garulli(at)orientechnologies.com) Version: pax-swissbox-extender-1.8.0.jar Copyright: Copyright 2007-2008 Alin Dreghiciu. diff --git a/openidm-doc/src/main/asciidoc/install-guide/chap-install.adoc b/openidm-doc/src/main/asciidoc/install-guide/chap-install.adoc index 23470f1377..1a6d04b132 100644 --- a/openidm-doc/src/main/asciidoc/install-guide/chap-install.adoc +++ b/openidm-doc/src/main/asciidoc/install-guide/chap-install.adoc @@ -558,14 +558,14 @@ When OpenIDM is running on the localhost system, you can access these UIs at `\h OpenIDM comes with an internal noSQL database, OrientDB, for use as the internal repository out of the box. This makes it easy to get started with OpenIDM. OrientDB is not supported for production use, however, so use a supported JDBC database when moving to production. -To query the internal noSQL database, download and extract link:https://search.maven.org/remotecontent?filepath=com/orientechnologies/orientdb-community/1.7.10/orientdb-community-1.7.10-distribution.zip[OrientDB (version 1.7.10), window=\_blank]. You will find the shell console in the `bin` directory. Start the OrientDB console, using `console.sh` or `console.bat`, and connect to the running OpenIDM instance, with the `connect` command: +To query the internal noSQL database, download and extract link:https://search.maven.org/remotecontent?filepath=com/orientechnologies/orientdb-community/2.2.37/orientdb-community-2.2.37.zip[OrientDB (version 2.2.37), window=\_blank]. You will find the shell console in the `bin` directory. Start the OrientDB console, using `console.sh` or `console.bat`, and connect to the running OpenIDM instance, with the `connect` command: [source, console] ---- -$ cd /path/to/orientdb-community-1.7.10/bin +$ cd /path/to/orientdb-community-2.2.37/bin $ ./console.sh -OrientDB console v.1.7.10 (build @BUILD@) www.orientechnologies.com +OrientDB console v.2.2.37 (build @BUILD@) www.orientechnologies.com Type 'help' to display all the commands supported. Installing extensions for GREMLIN language v.2.5.0 diff --git a/openidm-repo-orientdb/pom.xml b/openidm-repo-orientdb/pom.xml index de42898cf1..90afed89d9 100644 --- a/openidm-repo-orientdb/pom.xml +++ b/openidm-repo-orientdb/pom.xml @@ -99,11 +99,6 @@ - - com.orientechnologies - orientdb-enterprise - ${orientdb.version} - net.java.dev.jna jna diff --git a/openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/Activator.java b/openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/Activator.java index 570e696d94..3cd6e409f3 100644 --- a/openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/Activator.java +++ b/openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/Activator.java @@ -59,6 +59,21 @@ public void start(BundleContext context) { bootConfig.put(OrientDBRepoService.CONFIG_PASSWORD, repoConfig.get(OrientDBRepoService.CONFIG_PASSWORD.toLowerCase()).getObject()); bootConfig.put(OrientDBRepoService.CONFIG_POOL_MIN_SIZE, repoConfig.get(OrientDBRepoService.CONFIG_POOL_MIN_SIZE.toLowerCase()).getObject()); bootConfig.put(OrientDBRepoService.CONFIG_POOL_MAX_SIZE, repoConfig.get(OrientDBRepoService.CONFIG_POOL_MAX_SIZE.toLowerCase()).getObject()); + // Forward the full DB schema (dbStructure) to the bootstrap repo so that + // all OrientDB classes/clusters/indexes are created during the first + // storage open. DBHelper.checkDB temporarily disables USE_WAL while + // setting up the schema, but USE_WAL is read at storage-open time, so + // the disable only takes effect on the very first open. If the + // bootstrap creates the storage with only the "config" class, the WAL + // writer is started and any later schema additions performed by the + // full-service activation pay the full O(N²) WAL flush cost. By + // creating the entire schema during bootstrap (when USE_WAL is + // genuinely off) the second getPool call simply finds every class + // already present and skips all expensive schema work. + Object dbStructure = repoConfig.get(OrientDBRepoService.CONFIG_DB_STRUCTURE.toLowerCase()).getObject(); + if (dbStructure != null) { + bootConfig.put(OrientDBRepoService.CONFIG_DB_STRUCTURE, dbStructure); + } // Init the bootstrap repo bootSvc = OrientDBRepoService.getRepoBootService(bootConfig); diff --git a/openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/DBHelper.java b/openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/DBHelper.java index 39bc1d42c3..9072bffd85 100644 --- a/openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/DBHelper.java +++ b/openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/DBHelper.java @@ -306,8 +306,42 @@ private static void warmUpPool(ODatabaseDocumentPool pool, String dbURL, String private static ODatabaseDocumentTx checkDB(String dbURL, String user, String password, JsonValue completeConfig) throws InvalidException { + // Disable WAL and per-operation fsyncs before opening or creating the storage. + // USE_WAL is read at storage-open time, so it must be changed here, before + // ODatabaseDocumentTx is constructed. With WAL enabled (the default) OrientDB + // calls fdatasync() from a background thread every WAL_COMMIT_TIMEOUT ms, and + // the WAL grows with every new cluster or index. On Linux/macOS this produces + // O(N²) I/O: creating the N-th class triggers a flush of the entire WAL that + // already contains records for the N-1 previous classes. With 30 schema classes + // the total wall-clock time exceeds five minutes. + // + // The dominant cost on Linux is actually + // STORAGE_MAKE_FULL_CHECKPOINT_AFTER_CLUSTER_CREATE (default true): OrientDB + // performs a full storage checkpoint — fsync()ing every cluster file and the + // WAL — after EACH addCluster() call. As clusters accumulate, each + // checkpoint takes longer (it scans/syncs all previously-created clusters), + // giving the classic O(N²) total time observed during schema setup. + // + // Disabling these flags for the schema setup is safe in both the new-DB and + // the existing-DB paths: getPool() is synchronized static, so only one thread + // at a time touches these process-global flags, and the single setup + // connection is the only live connection to this storage (the pool for this + // dbURL is not created until after checkDB() returns). The storage is closed + // cleanly afterwards — all dirty pages are flushed to disk by the disk cache — + // so the pool that re-opens it next gets a consistent store with the original + // settings restored. + boolean origSyncOnUpdate = OGlobalConfiguration.STORAGE_CONFIGURATION_SYNC_ON_UPDATE.getValueAsBoolean(); + boolean origWalSyncOnPageFlush = OGlobalConfiguration.WAL_SYNC_ON_PAGE_FLUSH.getValueAsBoolean(); + boolean origUseWal = OGlobalConfiguration.USE_WAL.getValueAsBoolean(); + boolean origCheckpointAfterClusterCreate = + OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_CLUSTER_CREATE.getValueAsBoolean(); + boolean origCheckpointAfterCreate = + OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_CREATE.getValueAsBoolean(); + boolean origCheckpointAfterOpen = + OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_OPEN.getValueAsBoolean(); + // TODO: Creation/opening of db may be not be necessary if we require this managed externally - ODatabaseDocumentTx db = new ODatabaseDocumentTx(dbURL); + ODatabaseDocumentTx db; // To add support for remote DB checking/creation one // would need to use OServerAdmin instead @@ -315,22 +349,67 @@ private static ODatabaseDocumentTx checkDB(String dbURL, String user, String pas // Local DB we can auto populate if (isLocalDB(dbURL) || isMemoryDB(dbURL)) { - if (db.exists()) { - logger.info("Using DB at {}", dbURL); - db.open(user, password); - populateSample(db, completeConfig); - } else { - logger.info("DB does not exist, creating {}", dbURL); - db.create(); - // Delete default admin user - OSecurity security = db.getMetadata().getSecurity(); - security.dropUser(OUser.ADMIN); - // Create new admin user with new username and password - security.createUser(user, password, security.getRole(ORole.ADMIN)); + // Suppress all sync-on-write overhead for the duration of schema setup. + // These flags MUST be changed BEFORE constructing ODatabaseDocumentTx: + // OLocalPaginatedStorage reads OGlobalConfiguration.USE_WAL in its + // constructor (which is invoked from `new ODatabaseDocumentTx(url)`) + // and caches the value for the lifetime of the storage. Setting + // USE_WAL=false after the storage is already constructed has no + // effect, leaving the WAL writer active and producing the O(N²) + // schema-creation slowdown observed on Linux/macOS CI runners. + // + // getPool() is synchronized static, so only one thread runs this + // code at a time; the temporary global-config changes cannot + // affect concurrent database operations. The single setup + // connection is the only live connection to this storage (the + // pool for this dbURL is not created until after checkDB() + // returns), so it is safe to disable WAL globally here. + OGlobalConfiguration.STORAGE_CONFIGURATION_SYNC_ON_UPDATE.setValue(false); + OGlobalConfiguration.WAL_SYNC_ON_PAGE_FLUSH.setValue(false); + OGlobalConfiguration.USE_WAL.setValue(false); + // The real O(N²) culprit on Linux: by default OrientDB performs a full + // storage checkpoint (fsync of every cluster file plus WAL flush) after + // EACH addCluster() invocation. Each checkpoint scans/syncs every + // previously-created cluster, so total cost is O(N²). Disable for the + // duration of schema setup; we'll do a single explicit flush by closing + // the storage cleanly when populateSample returns. + OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_CLUSTER_CREATE.setValue(false); + OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_CREATE.setValue(false); + OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_OPEN.setValue(false); + try { + db = new ODatabaseDocumentTx(dbURL); + // exists() does not open the storage; it only probes the + // on-disk database files. The storage engine has already + // been instantiated above with USE_WAL=false captured. + boolean exists = db.exists(); + + if (exists) { + logger.info("Using DB at {}", dbURL); + db.open(user, password); + } else { + logger.info("DB does not exist, creating {}", dbURL); + db.create(); + // Delete default admin user + OSecurity security = db.getMetadata().getSecurity(); + security.dropUser(OUser.ADMIN); + // Create new admin user with new username and password + security.createUser(user, password, security.getRole(ORole.ADMIN)); + } populateSample(db, completeConfig); + } finally { + OGlobalConfiguration.STORAGE_CONFIGURATION_SYNC_ON_UPDATE.setValue(origSyncOnUpdate); + OGlobalConfiguration.WAL_SYNC_ON_PAGE_FLUSH.setValue(origWalSyncOnPageFlush); + OGlobalConfiguration.USE_WAL.setValue(origUseWal); + OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_CLUSTER_CREATE + .setValue(origCheckpointAfterClusterCreate); + OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_CREATE + .setValue(origCheckpointAfterCreate); + OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_OPEN + .setValue(origCheckpointAfterOpen); } } else { logger.info("Using remote DB at {}", dbURL); + db = new ODatabaseDocumentTx(dbURL); } return db; } diff --git a/openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/EmbeddedOServerService.java b/openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/EmbeddedOServerService.java index f4038bda4c..00cafef129 100644 --- a/openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/EmbeddedOServerService.java +++ b/openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/EmbeddedOServerService.java @@ -241,7 +241,7 @@ protected OServerConfiguration getOrientDBConfig(JsonValue config) { if (dbURL == null || dbURL.isEmpty()) { dbURL = "plocal:" + IdentityServer.getFileForWorkingPath("db/openidm").getAbsolutePath(); } - File dbFolder = IdentityServer.getFileForWorkingPath(dbURL.split(":")[1]); + File dbFolder = IdentityServer.getFileForWorkingPath(dbURL.substring(dbURL.indexOf(':') + 1)); OServerStorageConfiguration storage2 = new OServerStorageConfiguration(); storage2.name = "openidm"; @@ -269,7 +269,8 @@ protected OServerConfiguration getOrientDBConfig(JsonValue config) { }; // OrientDB currently logs a warning if this is not set, // although it should be taking the setting from the config above instead. - System.setProperty("ORIENTDB_HOME", dbFolder.getAbsolutePath()); + // Point to the OpenIDM working directory so that config/security.json can be found there. + System.setProperty("ORIENTDB_HOME", IdentityServer.getFileForWorkingPath("").getAbsolutePath()); return configuration; } diff --git a/openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/OrientDBRepoService.java b/openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/OrientDBRepoService.java index 951265583e..036cb12cb9 100644 --- a/openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/OrientDBRepoService.java +++ b/openidm-repo-orientdb/src/main/java/org/forgerock/openidm/repo/orientdb/impl/OrientDBRepoService.java @@ -88,7 +88,6 @@ import com.orientechnologies.orient.core.index.OIndexException; import com.orientechnologies.orient.core.record.impl.ODocument; import com.orientechnologies.orient.core.storage.ORecordDuplicatedException; -import com.orientechnologies.orient.core.version.OSimpleVersion; /** * Repository service implementation using OrientDB @@ -418,7 +417,7 @@ public ResourceResponse delete(DeleteRequest request) throws ResourceException { throw new NotFoundException("Object does not exist for delete on: " + request.getResourcePath()); } - db.delete(existingDoc.getIdentity(), new OSimpleVersion(ver)); + db.delete(existingDoc.getIdentity(), ver); logger.debug("delete for id succeeded: {} revision: {}", localId, request.getRevision()); return DocumentUtil.toResource(existingDoc); } catch (ODatabaseException ex) { diff --git a/openidm-zip/pom.xml b/openidm-zip/pom.xml index 9d88ce6209..7644dc458f 100644 --- a/openidm-zip/pom.xml +++ b/openidm-zip/pom.xml @@ -557,6 +557,11 @@ javax.annotation-api 1.3.2 + + com.google.code.findbugs + jsr305 + 3.0.2 + @@ -770,10 +775,9 @@ orientdb-community ${orientdb.studio.version} zip - distribution false ${project.build.directory}/ - **/www/**,**/plugins/studio-*.zip + **/www/**,**/plugins/*studio*.zip false @@ -792,8 +796,8 @@ - - + + diff --git a/openidm-zip/src/main/resources/config/security.json b/openidm-zip/src/main/resources/config/security.json new file mode 100644 index 0000000000..4c4bd42dc7 --- /dev/null +++ b/openidm-zip/src/main/resources/config/security.json @@ -0,0 +1,6 @@ +{ + "security": { + "enabled": false, + "debug": false + } +} diff --git a/openidm-zip/src/main/resources/startup.bat b/openidm-zip/src/main/resources/startup.bat index f3bcc2cfe4..1937748a45 100755 --- a/openidm-zip/src/main/resources/startup.bat +++ b/openidm-zip/src/main/resources/startup.bat @@ -35,7 +35,7 @@ for /f "delims=.-_ tokens=1-2" %%v in ("%JAVA_VERSION%") do ( ) set "ADD_OPENS_ARGS=" if /I %JAVA_VERSION% GEQ 9 ( - set ADD_OPENS_ARGS=--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED + set ADD_OPENS_ARGS=--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED ) set "JPDA=" diff --git a/openidm-zip/src/main/resources/startup.sh b/openidm-zip/src/main/resources/startup.sh index 3e9bb2974c..38771b77c6 100755 --- a/openidm-zip/src/main/resources/startup.sh +++ b/openidm-zip/src/main/resources/startup.sh @@ -43,6 +43,7 @@ if [ $JAVA_VER -ge 90 ]; then --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED" fi diff --git a/pom.xml b/pom.xml index 15f42c57b1..9d6755b34a 100644 --- a/pom.xml +++ b/pom.xml @@ -115,9 +115,9 @@ + 2.2.37 + 2.2.37 3.5.16 - 2.1.25 - 1.7.10 2.0.0-alpha-1 2.9.4 2.2.220 @@ -184,6 +184,7 @@ --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.security=ALL-UNNAMED + --add-opens java.base/sun.nio.ch=ALL-UNNAMED --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED