The Unified and Autonomous Development Platform
Ignore resource accessing warnings
robin shen129a6f00eeeb
diff --git a/server-core/src/main/java/io/onedev/server/persistence/DefaultPersistManager.java b/server-core/src/main/java/io/onedev/server/persistence/DefaultPersistManager.javaindex dc4c1c7881..5212e816a9 100644--- a/server-core/src/main/java/io/onedev/server/persistence/DefaultPersistManager.java+++ b/server-core/src/main/java/io/onedev/server/persistence/DefaultPersistManager.java@@ -1,614 +1,616 @@-package io.onedev.server.persistence;--import java.io.File;-import java.io.FilenameFilter;-import java.io.IOException;-import java.lang.reflect.Field;-import java.nio.charset.Charset;-import java.sql.Connection;-import java.sql.Driver;-import java.sql.ResultSet;-import java.sql.SQLException;-import java.sql.Statement;-import java.util.ArrayList;-import java.util.Collections;-import java.util.EnumSet;-import java.util.List;-import java.util.Properties;--import javax.persistence.EntityManagerFactory;-import javax.persistence.ManyToOne;-import javax.persistence.criteria.CriteriaBuilder;-import javax.persistence.criteria.CriteriaQuery;-import javax.persistence.criteria.Root;-import javax.persistence.metamodel.EntityType;--import org.dom4j.DocumentHelper;-import org.dom4j.Element;-import org.hibernate.Interceptor;-import org.hibernate.ReplicationMode;-import org.hibernate.Session;-import org.hibernate.SessionFactory;-import org.hibernate.Transaction;-import org.hibernate.boot.Metadata;-import org.hibernate.boot.MetadataBuilder;-import org.hibernate.boot.MetadataSources;-import org.hibernate.boot.model.naming.Identifier;-import org.hibernate.boot.model.naming.PhysicalNamingStrategy;-import org.hibernate.boot.registry.StandardServiceRegistry;-import org.hibernate.boot.registry.StandardServiceRegistryBuilder;-import org.hibernate.engine.jdbc.env.spi.JdbcEnvironment;-import org.hibernate.query.Query;-import org.hibernate.tool.hbm2ddl.SchemaExport;-import org.hibernate.tool.schema.TargetType;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--import com.google.common.base.Preconditions;-import com.google.inject.Inject;-import com.google.inject.Singleton;--import io.onedev.commons.utils.ClassUtils;-import io.onedev.commons.utils.ExceptionUtils;-import io.onedev.commons.utils.FileUtils;-import io.onedev.server.migration.DataMigrator;-import io.onedev.server.migration.MigrationHelper;-import io.onedev.server.migration.VersionedDocument;-import io.onedev.server.model.AbstractEntity;-import io.onedev.server.model.ModelVersion;-import io.onedev.server.persistence.annotation.Sessional;-import io.onedev.server.persistence.dao.Dao;-import io.onedev.server.util.BeanUtils;-import io.onedev.server.util.validation.EntityValidator;--@Singleton-public class DefaultPersistManager implements PersistManager {-- private static final int BACKUP_BATCH_SIZE = 1000;-- private static final Logger logger = LoggerFactory.getLogger(DefaultPersistManager.class);-- protected final PhysicalNamingStrategy physicalNamingStrategy;-- protected final HibernateProperties properties;-- protected final Interceptor interceptor;-- protected final IdManager idManager;-- protected final Dao dao;-- protected final EntityValidator validator;-- protected final StandardServiceRegistry serviceRegistry;-- protected final TransactionManager transactionManager;-- protected volatile SessionFactory sessionFactory;-- @Inject- public DefaultPersistManager(PhysicalNamingStrategy physicalNamingStrategy,- HibernateProperties properties, Interceptor interceptor,- IdManager idManager, Dao dao, EntityValidator validator,- TransactionManager transactionManager) {- this.physicalNamingStrategy = physicalNamingStrategy;- this.properties = properties;- this.interceptor = interceptor;- this.idManager = idManager;- this.dao = dao;- this.validator = validator;- this.transactionManager = transactionManager;- serviceRegistry = new StandardServiceRegistryBuilder().applySettings(properties).build();- }-- protected String getDialect() {- return properties.getDialect();- }-- protected void execute(List<String> sqls, boolean failOnError) {- try ( Connection conn = getConnection();- Statement stmt = conn.createStatement();) {- for (String sql: sqls) {- logger.trace("Executing sql: " + sql);- if (failOnError) {- stmt.execute(sql);- } else {- try {- stmt.execute(sql);- } catch (Exception e) {- logger.error("Error executing sql", e);- }- }- }- } catch (SQLException e) {- throw new RuntimeException(e);- }- }-- protected Metadata buildMetadata() {- MetadataSources metadataSources = new MetadataSources(serviceRegistry);- for (Class<? extends AbstractEntity> each: ClassUtils.findImplementations(AbstractEntity.class, AbstractEntity.class)) {- metadataSources.addAnnotatedClass(each);- }-- MetadataBuilder builder = metadataSources.getMetadataBuilder();- builder.applyPhysicalNamingStrategy(physicalNamingStrategy);- return builder.build();- }-- protected SessionFactory buildSessionFactory(Metadata metadata) {- return metadata.getSessionFactoryBuilder().applyInterceptor(interceptor).build();- }-- protected String checkDataVersion(boolean allowEmptyDB) {- String dbDataVersion = readDbDataVersion();- if (!allowEmptyDB && dbDataVersion == null) {- logger.error("Database is not populated yet");- System.exit(1);- }- String appDataVersion = MigrationHelper.getVersion(DataMigrator.class);- if (dbDataVersion != null && !dbDataVersion.equals(appDataVersion)) {- logger.error("Data version mismatch (app data version: {}, db data version: {})", appDataVersion, dbDataVersion);- System.exit(1);- }- return dbDataVersion;- }-- @Override- public void start() {- String dbDataVersion = checkDataVersion(true);-- Metadata metadata = buildMetadata();-- if (dbDataVersion == null) {- File tempFile = null;- try {- tempFile = File.createTempFile("schema", ".sql");- new SchemaExport().setOutputFile(tempFile.getAbsolutePath())- .setFormat(false).createOnly(EnumSet.of(TargetType.SCRIPT), metadata);- List<String> sqls = new ArrayList<String>();- for (String sql: FileUtils.readLines(tempFile, Charset.defaultCharset())) {- if (shouldInclude(sql))- sqls.add(sql);- }- execute(sqls, true);- } catch (IOException e) {- throw new RuntimeException(e);- } finally {- if (tempFile != null)- tempFile.delete();- }-- sessionFactory = buildSessionFactory(metadata);-- idManager.init();-- transactionManager.run(new Runnable() {-- @Override- public void run() {- ModelVersion dataVersion = new ModelVersion();- dataVersion.versionColumn = MigrationHelper.getVersion(DataMigrator.class);- transactionManager.getSession().save(dataVersion);- }-- });- } else {- sessionFactory = metadata.getSessionFactoryBuilder().applyInterceptor(interceptor).build();- idManager.init();- }- }-- @Override- public void stop() {- if (sessionFactory != null)- sessionFactory.close();- }-- protected Connection getConnection() {- try {- Driver driver = (Driver) Class.forName(properties.getDriver(), true,- Thread.currentThread().getContextClassLoader()).newInstance();- Properties connectProps = new Properties();- String user = properties.getUser();- String password = properties.getPassword();- if (user != null)- connectProps.put("user", user);- if (password != null)- connectProps.put("password", password);-- return driver.connect(properties.getUrl(), connectProps);- } catch (Exception e) {- throw ExceptionUtils.unchecked(e);- }- }-- private String getVersionTableName() {- JdbcEnvironment environment = serviceRegistry.getService(JdbcEnvironment.class);- Identifier identifier = Identifier.toIdentifier(ModelVersion.class.getSimpleName());- return physicalNamingStrategy.toPhysicalTableName(identifier, environment).getText();- }-- private String getVersionFieldName() {- return ModelVersion.class.getFields()[0].getName();- }-- private String getVersionColumnName() {- JdbcEnvironment environment = serviceRegistry.getService(JdbcEnvironment.class);- Identifier identifier = Identifier.toIdentifier(getVersionFieldName());- return physicalNamingStrategy.toPhysicalColumnName(identifier, environment).getText();- }-- protected String readDbDataVersion() {- try (Connection conn = getConnection()) {- conn.setAutoCommit(false);- String versionTableName = getVersionTableName();- boolean versionTableExists = false;- try (ResultSet resultset = conn.getMetaData().getTables(null, null, versionTableName.toUpperCase(), null)) {- if (resultset.next())- versionTableExists = true;- }- if (!versionTableExists) {- try (ResultSet resultset = conn.getMetaData().getTables(null, null, versionTableName.toLowerCase(), null)) {- if (resultset.next()) {- versionTableExists = true;- }- }- }- if (!versionTableExists) {- try (ResultSet resultset = conn.getMetaData().getTables(null, null, versionTableName, null)) {- if (resultset.next()) {- versionTableExists = true;- }- }- }- if (!versionTableExists) {- return null;- } else {- try ( Statement stmt = conn.createStatement();- ResultSet resultset = stmt.executeQuery("select " + getVersionColumnName() + " from " + versionTableName)) {- if (!resultset.next())- throw new RuntimeException("No data version found in database: this is normally caused "- + "by unsuccessful restore/upgrade, please clean the database and try again");- String dataVersion = resultset.getString(1);- if (resultset.next())- throw new RuntimeException("Illegal data version format in database");- return dataVersion;- }- }- } catch (Exception e) {- if (e.getMessage() != null && e.getMessage().contains("ORA-00942"))- return null;- else- throw ExceptionUtils.unchecked(e);- }- }-- private File getVersionFile(File dataDir) {- File versionFile = new File(dataDir, ModelVersion.class.getSimpleName() + "s.xml");- if (!versionFile.exists())- versionFile = new File(dataDir, "VersionTables.xml");- return versionFile;- }-- protected void migrateData(File dataDir) {- File versionFile = getVersionFile(dataDir);-- VersionedDocument dom = VersionedDocument.fromFile(versionFile);- List<Element> elements = dom.getRootElement().elements();- if (elements.size() != 1)- throw new RuntimeException("Incorrect data format: illegal data version");- Element versionElement = elements.iterator().next().element(getVersionFieldName());- if (versionElement == null) {- throw new RuntimeException("Incorrect data format: no data version");- }-- if (MigrationHelper.migrate(versionElement.getText(), new DataMigrator(), dataDir)) {- // load version file again in case we changed something of it while migrating- versionFile = getVersionFile(dataDir);- dom = VersionedDocument.fromFile(versionFile);- elements = dom.getRootElement().elements();- Preconditions.checkState(elements.size() == 1);- versionElement = Preconditions.checkNotNull(elements.iterator().next().element(getVersionFieldName()));- versionElement.setText(MigrationHelper.getVersion(DataMigrator.class));- dom.writeToFile(versionFile, false);- }- }-- /**- * Determines whether or not entityType1 has transitive foreign key- * dependency on entityType2.- *- * @param entityType1- * @param entityType2- * @return- */- private boolean hasForeignKeyDependency(Class<?> entityType1, Class<?> entityType2) {- for (Field field: BeanUtils.findFields(entityType1)) {- if (field.getAnnotation(ManyToOne.class) != null) {- if (field.getType() == entityType2)- return true;- if (field.getType() != entityType1 &&- hasForeignKeyDependency(field.getType(), entityType2)) {- return true;- }- }- }-- return false;- }-- /**- * @return hibernate entity types ordered using foreign key dependency- * information. For example, Build type comes before Project- * type as Build has foreign key reference to Project- */- private List<Class<?>> getEntityTypes(SessionFactory sessionFactory) {- List<Class<?>> entityTypes = new ArrayList<>();- for (EntityType<?> entityType: ((EntityManagerFactory)sessionFactory).getMetamodel().getEntities()) {- entityTypes.add(entityType.getJavaType());- }-- /* Collections.sort does not work here */- List<Class<?>> sorted = new ArrayList<Class<?>>();- while (!entityTypes.isEmpty()) {- Class<?> dependencyLeaf = null;- for (Class<?> entityType: entityTypes) {- boolean hasDependents = false;- for (Class<?> each: entityTypes) {- if (each != entityType) {- if (hasForeignKeyDependency(each, entityType)) {- hasDependents = true;- break;- }- }- }- if (!hasDependents) {- dependencyLeaf = entityType;- break;- }- }- if (dependencyLeaf != null) {- sorted.add(dependencyLeaf);- entityTypes.remove(dependencyLeaf);- } else {- throw new RuntimeException("Looped foreigh key dependency found between model classes");- }- }- return sorted;- }-- @Override- public void exportData(File exportDir) {- exportData(exportDir, BACKUP_BATCH_SIZE);- }-- @Sessional- @Override- public void exportData(File exportDir, int batchSize) {- Session session = sessionFactory.openSession();- for (Class<?> entityType: getEntityTypes(sessionFactory)) {- logger.info("Exporting table '" + entityType.getSimpleName() + "'...");-- logger.info("Querying table ids...");-- CriteriaBuilder builder = sessionFactory.getCriteriaBuilder();- CriteriaQuery<Number> query = builder.createQuery(Number.class);- Root<?> root = query.from(entityType);- query.select(root.get("id")).orderBy(builder.asc(root.get("id")));-- List<Number> ids = session.createQuery(query).list();-- int count = ids.size();-- for (int i=0; i<count/batchSize; i++) {- exportEntity(session, entityType, ids, i*batchSize, batchSize, batchSize, exportDir);- // clear session to free memory- session.clear();- }-- if (count%batchSize != 0) {- exportEntity(session, entityType, ids, count/batchSize*batchSize, count%batchSize, batchSize, exportDir);- }- logger.info("");- }- }-- private void exportEntity(Session session, Class<?> entityType, List<Number> ids, int start, int count, int batchSize, File exportDir) {- logger.info("Loading table rows ({}->{}) from database...", String.valueOf(start+1), (start + count));-- Query<?> query = session.createQuery("from " + entityType.getSimpleName() + " where id>=:fromId and id<=:toId");- query.setParameter("fromId", ids.get(start));- query.setParameter("toId", ids.get(start+count-1));-- logger.info("Converting table rows to XML...");- VersionedDocument dom = new VersionedDocument();- Element rootElement = dom.addElement("list");- for (Object entity: query.list())- rootElement.appendContent(VersionedDocument.fromBean(entity));- String fileName;-- if (start == 0)- fileName = entityType.getSimpleName() + "s.xml";- else- fileName = entityType.getSimpleName() + "s.xml." + (start/batchSize + 1);-- logger.info("Writing resulting XML to file '" + fileName + "...");- dom.writeToFile(new File(exportDir, fileName), true);- }-- /*- * We do not use @Transactional annotation and will manage the session and transaction manually- * in this method to reduce memory usage if importing a large database.- */- @Sessional- @Override- public void importData(Metadata metadata, File dataDir) {- Session session = dao.getSession();- List<Class<?>> entityTypes = getEntityTypes(sessionFactory);- Collections.reverse(entityTypes);- for (Class<?> entityType: entityTypes) {- File[] dataFiles = dataDir.listFiles(new FilenameFilter() {-- @Override- public boolean accept(File dir, String name) {- return name.startsWith(entityType.getSimpleName() + "s.xml");- }-- });- for (File file: dataFiles) {- Transaction transaction = session.beginTransaction();- try {- logger.info("Importing from data file '" + file.getName() + "'...");- VersionedDocument dom = VersionedDocument.fromFile(file);-- for (Element element: dom.getRootElement().elements()) {- element.detach();- AbstractEntity entity = (AbstractEntity) new VersionedDocument(DocumentHelper.createDocument(element)).toBean();- session.replicate(entity, ReplicationMode.EXCEPTION);- }- session.flush();- session.clear();- transaction.commit();- } catch (Exception e) {- transaction.rollback();- throw ExceptionUtils.unchecked(e);- }- }- }- }-- protected void validateData(Metadata metadata, File dataDir) {- List<Class<?>> entityTypes = getEntityTypes(sessionFactory);- Collections.reverse(entityTypes);- for (Class<?> entityType: entityTypes) {- File[] dataFiles = dataDir.listFiles(new FilenameFilter() {-- @Override- public boolean accept(File dir, String name) {- return name.startsWith(entityType.getSimpleName() + "s.xml");- }-- });- for (File file: dataFiles) {- try {- logger.info("Validating data file '" + file.getName() + "'...");- VersionedDocument dom = VersionedDocument.fromFile(file);-- for (Element element: dom.getRootElement().elements()) {- element.detach();- AbstractEntity entity = (AbstractEntity) new VersionedDocument(DocumentHelper.createDocument(element)).toBean();- validator.validate(entity);- }- } catch (Exception e) {- throw ExceptionUtils.unchecked(e);- }- }- }- }-- protected void applyConstraints(Metadata metadata) {- File tempFile = null;- try {- tempFile = File.createTempFile("schema", ".sql");- new SchemaExport().setOutputFile(tempFile.getAbsolutePath())- .setFormat(false).createOnly(EnumSet.of(TargetType.SCRIPT), metadata);- List<String> sqls = new ArrayList<>();- for (String sql: FileUtils.readLines(tempFile, Charset.defaultCharset())) {- if (isApplyingConstraints(sql)) {- sqls.add(sql);- }- }- execute(sqls, true);- } catch (IOException e) {- throw new RuntimeException(e);- } finally {- if (tempFile != null)- tempFile.delete();- }- }-- protected void createTables(Metadata metadata) {- File tempFile = null;- try {- tempFile = File.createTempFile("schema", ".sql");- new SchemaExport().setOutputFile(tempFile.getAbsolutePath())- .setFormat(false).createOnly(EnumSet.of(TargetType.SCRIPT), metadata);- List<String> sqls = new ArrayList<>();- for (String sql: FileUtils.readLines(tempFile, Charset.defaultCharset())) {- if (shouldInclude(sql) && !isApplyingConstraints(sql))- sqls.add(sql);- }- execute(sqls, true);- } catch (IOException e) {- throw new RuntimeException(e);- } finally {- if (tempFile != null)- FileUtils.deleteFile(tempFile);- }- }-- protected void dropConstraints(Metadata metadata) {- File tempFile = null;- try {- tempFile = File.createTempFile("schema", ".sql");- new SchemaExport().setOutputFile(tempFile.getAbsolutePath())- .setFormat(false).drop(EnumSet.of(TargetType.SCRIPT), metadata);- List<String> sqls = new ArrayList<>();- for (String sql: FileUtils.readLines(tempFile, Charset.defaultCharset())) {- if (isDroppingConstraints(sql))- sqls.add(sql);- }- execute(sqls, false);- } catch (IOException e) {- throw new RuntimeException(e);- } finally {- if (tempFile != null)- tempFile.delete();- }- }-- protected void cleanDatabase(Metadata metadata) {- File tempFile = null;- try {- tempFile = File.createTempFile("schema", ".sql");- new SchemaExport().setOutputFile(tempFile.getAbsolutePath())- .setFormat(false).drop(EnumSet.of(TargetType.SCRIPT), metadata);- List<String> sqls = new ArrayList<>();- for (String sql: FileUtils.readLines(tempFile, Charset.defaultCharset())) {- sqls.add(sql);- }- execute(sqls, false);- } catch (IOException e) {- throw new RuntimeException(e);- } finally {- if (tempFile != null)- tempFile.delete();- }- }-- private boolean isApplyingConstraints(String sql) {- return sql.toLowerCase().contains(" foreign key ");- }-- private boolean isDroppingConstraints(String sql) {- return sql.toLowerCase().contains(" drop constraint ");- }-- private boolean shouldInclude(String sql) {- // some databases will create index on foreign keys automatically, so- // we skip creating indexes for those- if (!sql.toLowerCase().startsWith("create index") || !sql.toLowerCase().endsWith("_id)")) {- return true;- } else {- String dialect = getDialect().toLowerCase();- return !dialect.contains("mysql") && !dialect.contains("hsql");- }- }-- @Override- public SessionFactory getSessionFactory() {- return sessionFactory;- }--}+package io.onedev.server.persistence;++import java.io.File;+import java.io.FilenameFilter;+import java.io.IOException;+import java.lang.reflect.Field;+import java.nio.charset.Charset;+import java.sql.Connection;+import java.sql.Driver;+import java.sql.ResultSet;+import java.sql.SQLException;+import java.sql.Statement;+import java.util.ArrayList;+import java.util.Collections;+import java.util.EnumSet;+import java.util.List;+import java.util.Properties;++import javax.persistence.EntityManagerFactory;+import javax.persistence.ManyToOne;+import javax.persistence.criteria.CriteriaBuilder;+import javax.persistence.criteria.CriteriaQuery;+import javax.persistence.criteria.Root;+import javax.persistence.metamodel.EntityType;++import org.dom4j.DocumentHelper;+import org.dom4j.Element;+import org.hibernate.Interceptor;+import org.hibernate.ReplicationMode;+import org.hibernate.Session;+import org.hibernate.SessionFactory;+import org.hibernate.Transaction;+import org.hibernate.boot.Metadata;+import org.hibernate.boot.MetadataBuilder;+import org.hibernate.boot.MetadataSources;+import org.hibernate.boot.model.naming.Identifier;+import org.hibernate.boot.model.naming.PhysicalNamingStrategy;+import org.hibernate.boot.registry.StandardServiceRegistry;+import org.hibernate.boot.registry.StandardServiceRegistryBuilder;+import org.hibernate.engine.jdbc.env.spi.JdbcEnvironment;+import org.hibernate.query.Query;+import org.hibernate.tool.hbm2ddl.SchemaExport;+import org.hibernate.tool.schema.TargetType;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++import com.google.common.base.Preconditions;+import com.google.inject.Inject;+import com.google.inject.Singleton;++import io.onedev.commons.utils.ClassUtils;+import io.onedev.commons.utils.ExceptionUtils;+import io.onedev.commons.utils.FileUtils;+import io.onedev.server.migration.DataMigrator;+import io.onedev.server.migration.MigrationHelper;+import io.onedev.server.migration.VersionedDocument;+import io.onedev.server.model.AbstractEntity;+import io.onedev.server.model.ModelVersion;+import io.onedev.server.persistence.annotation.Sessional;+import io.onedev.server.persistence.dao.Dao;+import io.onedev.server.util.BeanUtils;+import io.onedev.server.util.validation.EntityValidator;++@Singleton+public class DefaultPersistManager implements PersistManager {++ private static final int BACKUP_BATCH_SIZE = 1000;++ private static final Logger logger = LoggerFactory.getLogger(DefaultPersistManager.class);++ protected final PhysicalNamingStrategy physicalNamingStrategy;++ protected final HibernateProperties properties;++ protected final Interceptor interceptor;++ protected final IdManager idManager;++ protected final Dao dao;++ protected final EntityValidator validator;++ protected final StandardServiceRegistry serviceRegistry;++ protected final TransactionManager transactionManager;++ protected volatile SessionFactory sessionFactory;++ @Inject+ public DefaultPersistManager(PhysicalNamingStrategy physicalNamingStrategy,+ HibernateProperties properties, Interceptor interceptor,+ IdManager idManager, Dao dao, EntityValidator validator,+ TransactionManager transactionManager) {+ this.physicalNamingStrategy = physicalNamingStrategy;+ this.properties = properties;+ this.interceptor = interceptor;+ this.idManager = idManager;+ this.dao = dao;+ this.validator = validator;+ this.transactionManager = transactionManager;+ serviceRegistry = new StandardServiceRegistryBuilder().applySettings(properties).build();+ }++ protected String getDialect() {+ return properties.getDialect();+ }++ protected void execute(List<String> sqls, boolean failOnError) {+ try ( Connection conn = getConnection();+ Statement stmt = conn.createStatement();) {+ for (String sql: sqls) {+ logger.trace("Executing sql: " + sql);+ if (failOnError) {+ stmt.execute(sql);+ } else {+ try {+ stmt.execute(sql);+ } catch (Exception e) {+ logger.error("Error executing sql", e);+ }+ }+ }+ } catch (SQLException e) {+ throw new RuntimeException(e);+ }+ }++ protected Metadata buildMetadata() {+ MetadataSources metadataSources = new MetadataSources(serviceRegistry);+ for (Class<? extends AbstractEntity> each: ClassUtils.findImplementations(AbstractEntity.class, AbstractEntity.class)) {+ metadataSources.addAnnotatedClass(each);+ }++ MetadataBuilder builder = metadataSources.getMetadataBuilder();+ builder.applyPhysicalNamingStrategy(physicalNamingStrategy);+ return builder.build();+ }++ protected SessionFactory buildSessionFactory(Metadata metadata) {+ return metadata.getSessionFactoryBuilder().applyInterceptor(interceptor).build();+ }++ protected String checkDataVersion(boolean allowEmptyDB) {+ String dbDataVersion = readDbDataVersion();+ if (!allowEmptyDB && dbDataVersion == null) {+ logger.error("Database is not populated yet");+ System.exit(1);+ }+ String appDataVersion = MigrationHelper.getVersion(DataMigrator.class);+ if (dbDataVersion != null && !dbDataVersion.equals(appDataVersion)) {+ logger.error("Data version mismatch (app data version: {}, db data version: {})", appDataVersion, dbDataVersion);+ System.exit(1);+ }+ return dbDataVersion;+ }++ @Override+ public void start() {+ String dbDataVersion = checkDataVersion(true);++ Metadata metadata = buildMetadata();++ if (dbDataVersion == null) {+ File tempFile = null;+ try {+ tempFile = File.createTempFile("schema", ".sql");+ new SchemaExport().setOutputFile(tempFile.getAbsolutePath())+ .setFormat(false).createOnly(EnumSet.of(TargetType.SCRIPT), metadata);+ List<String> sqls = new ArrayList<String>();+ for (String sql: FileUtils.readLines(tempFile, Charset.defaultCharset())) {+ if (shouldInclude(sql))+ sqls.add(sql);+ }+ execute(sqls, true);+ } catch (IOException e) {+ throw new RuntimeException(e);+ } finally {+ if (tempFile != null)+ tempFile.delete();+ }++ sessionFactory = buildSessionFactory(metadata);++ idManager.init();++ transactionManager.run(new Runnable() {++ @Override+ public void run() {+ ModelVersion dataVersion = new ModelVersion();+ dataVersion.versionColumn = MigrationHelper.getVersion(DataMigrator.class);+ transactionManager.getSession().save(dataVersion);+ }++ });+ } else {+ sessionFactory = metadata.getSessionFactoryBuilder().applyInterceptor(interceptor).build();+ idManager.init();+ }+ }++ @Override+ public void stop() {+ if (sessionFactory != null) {+ sessionFactory.close();+ sessionFactory = null;+ }+ }++ protected Connection getConnection() {+ try {+ Driver driver = (Driver) Class.forName(properties.getDriver(), true,+ Thread.currentThread().getContextClassLoader()).newInstance();+ Properties connectProps = new Properties();+ String user = properties.getUser();+ String password = properties.getPassword();+ if (user != null)+ connectProps.put("user", user);+ if (password != null)+ connectProps.put("password", password);++ return driver.connect(properties.getUrl(), connectProps);+ } catch (Exception e) {+ throw ExceptionUtils.unchecked(e);+ }+ }++ private String getVersionTableName() {+ JdbcEnvironment environment = serviceRegistry.getService(JdbcEnvironment.class);+ Identifier identifier = Identifier.toIdentifier(ModelVersion.class.getSimpleName());+ return physicalNamingStrategy.toPhysicalTableName(identifier, environment).getText();+ }++ private String getVersionFieldName() {+ return ModelVersion.class.getFields()[0].getName();+ }++ private String getVersionColumnName() {+ JdbcEnvironment environment = serviceRegistry.getService(JdbcEnvironment.class);+ Identifier identifier = Identifier.toIdentifier(getVersionFieldName());+ return physicalNamingStrategy.toPhysicalColumnName(identifier, environment).getText();+ }++ protected String readDbDataVersion() {+ try (Connection conn = getConnection()) {+ conn.setAutoCommit(false);+ String versionTableName = getVersionTableName();+ boolean versionTableExists = false;+ try (ResultSet resultset = conn.getMetaData().getTables(null, null, versionTableName.toUpperCase(), null)) {+ if (resultset.next())+ versionTableExists = true;+ }+ if (!versionTableExists) {+ try (ResultSet resultset = conn.getMetaData().getTables(null, null, versionTableName.toLowerCase(), null)) {+ if (resultset.next()) {+ versionTableExists = true;+ }+ }+ }+ if (!versionTableExists) {+ try (ResultSet resultset = conn.getMetaData().getTables(null, null, versionTableName, null)) {+ if (resultset.next()) {+ versionTableExists = true;+ }+ }+ }+ if (!versionTableExists) {+ return null;+ } else {+ try ( Statement stmt = conn.createStatement();+ ResultSet resultset = stmt.executeQuery("select " + getVersionColumnName() + " from " + versionTableName)) {+ if (!resultset.next())+ throw new RuntimeException("No data version found in database: this is normally caused "+ + "by unsuccessful restore/upgrade, please clean the database and try again");+ String dataVersion = resultset.getString(1);+ if (resultset.next())+ throw new RuntimeException("Illegal data version format in database");+ return dataVersion;+ }+ }+ } catch (Exception e) {+ if (e.getMessage() != null && e.getMessage().contains("ORA-00942"))+ return null;+ else+ throw ExceptionUtils.unchecked(e);+ }+ }++ private File getVersionFile(File dataDir) {+ File versionFile = new File(dataDir, ModelVersion.class.getSimpleName() + "s.xml");+ if (!versionFile.exists())+ versionFile = new File(dataDir, "VersionTables.xml");+ return versionFile;+ }++ protected void migrateData(File dataDir) {+ File versionFile = getVersionFile(dataDir);++ VersionedDocument dom = VersionedDocument.fromFile(versionFile);+ List<Element> elements = dom.getRootElement().elements();+ if (elements.size() != 1)+ throw new RuntimeException("Incorrect data format: illegal data version");+ Element versionElement = elements.iterator().next().element(getVersionFieldName());+ if (versionElement == null) {+ throw new RuntimeException("Incorrect data format: no data version");+ }++ if (MigrationHelper.migrate(versionElement.getText(), new DataMigrator(), dataDir)) {+ // load version file again in case we changed something of it while migrating+ versionFile = getVersionFile(dataDir);+ dom = VersionedDocument.fromFile(versionFile);+ elements = dom.getRootElement().elements();+ Preconditions.checkState(elements.size() == 1);+ versionElement = Preconditions.checkNotNull(elements.iterator().next().element(getVersionFieldName()));+ versionElement.setText(MigrationHelper.getVersion(DataMigrator.class));+ dom.writeToFile(versionFile, false);+ }+ }++ /**+ * Determines whether or not entityType1 has transitive foreign key+ * dependency on entityType2.+ *+ * @param entityType1+ * @param entityType2+ * @return+ */+ private boolean hasForeignKeyDependency(Class<?> entityType1, Class<?> entityType2) {+ for (Field field: BeanUtils.findFields(entityType1)) {+ if (field.getAnnotation(ManyToOne.class) != null) {+ if (field.getType() == entityType2)+ return true;+ if (field.getType() != entityType1 &&+ hasForeignKeyDependency(field.getType(), entityType2)) {+ return true;+ }+ }+ }++ return false;+ }++ /**+ * @return hibernate entity types ordered using foreign key dependency+ * information. For example, Build type comes before Project+ * type as Build has foreign key reference to Project+ */+ private List<Class<?>> getEntityTypes(SessionFactory sessionFactory) {+ List<Class<?>> entityTypes = new ArrayList<>();+ for (EntityType<?> entityType: ((EntityManagerFactory)sessionFactory).getMetamodel().getEntities()) {+ entityTypes.add(entityType.getJavaType());+ }++ /* Collections.sort does not work here */+ List<Class<?>> sorted = new ArrayList<Class<?>>();+ while (!entityTypes.isEmpty()) {+ Class<?> dependencyLeaf = null;+ for (Class<?> entityType: entityTypes) {+ boolean hasDependents = false;+ for (Class<?> each: entityTypes) {+ if (each != entityType) {+ if (hasForeignKeyDependency(each, entityType)) {+ hasDependents = true;+ break;+ }+ }+ }+ if (!hasDependents) {+ dependencyLeaf = entityType;+ break;+ }+ }+ if (dependencyLeaf != null) {+ sorted.add(dependencyLeaf);+ entityTypes.remove(dependencyLeaf);+ } else {+ throw new RuntimeException("Looped foreigh key dependency found between model classes");+ }+ }+ return sorted;+ }++ @Override+ public void exportData(File exportDir) {+ exportData(exportDir, BACKUP_BATCH_SIZE);+ }++ @Sessional+ @Override+ public void exportData(File exportDir, int batchSize) {+ Session session = sessionFactory.openSession();+ for (Class<?> entityType: getEntityTypes(sessionFactory)) {+ logger.info("Exporting table '" + entityType.getSimpleName() + "'...");++ logger.info("Querying table ids...");++ CriteriaBuilder builder = sessionFactory.getCriteriaBuilder();+ CriteriaQuery<Number> query = builder.createQuery(Number.class);+ Root<?> root = query.from(entityType);+ query.select(root.get("id")).orderBy(builder.asc(root.get("id")));++ List<Number> ids = session.createQuery(query).list();++ int count = ids.size();++ for (int i=0; i<count/batchSize; i++) {+ exportEntity(session, entityType, ids, i*batchSize, batchSize, batchSize, exportDir);+ // clear session to free memory+ session.clear();+ }++ if (count%batchSize != 0) {+ exportEntity(session, entityType, ids, count/batchSize*batchSize, count%batchSize, batchSize, exportDir);+ }+ logger.info("");+ }+ }++ private void exportEntity(Session session, Class<?> entityType, List<Number> ids, int start, int count, int batchSize, File exportDir) {+ logger.info("Loading table rows ({}->{}) from database...", String.valueOf(start+1), (start + count));++ Query<?> query = session.createQuery("from " + entityType.getSimpleName() + " where id>=:fromId and id<=:toId");+ query.setParameter("fromId", ids.get(start));+ query.setParameter("toId", ids.get(start+count-1));++ logger.info("Converting table rows to XML...");+ VersionedDocument dom = new VersionedDocument();+ Element rootElement = dom.addElement("list");+ for (Object entity: query.list())+ rootElement.appendContent(VersionedDocument.fromBean(entity));+ String fileName;++ if (start == 0)+ fileName = entityType.getSimpleName() + "s.xml";+ else+ fileName = entityType.getSimpleName() + "s.xml." + (start/batchSize + 1);++ logger.info("Writing resulting XML to file '" + fileName + "...");+ dom.writeToFile(new File(exportDir, fileName), true);+ }++ /*+ * We do not use @Transactional annotation and will manage the session and transaction manually+ * in this method to reduce memory usage if importing a large database.+ */+ @Sessional+ @Override+ public void importData(Metadata metadata, File dataDir) {+ Session session = dao.getSession();+ List<Class<?>> entityTypes = getEntityTypes(sessionFactory);+ Collections.reverse(entityTypes);+ for (Class<?> entityType: entityTypes) {+ File[] dataFiles = dataDir.listFiles(new FilenameFilter() {++ @Override+ public boolean accept(File dir, String name) {+ return name.startsWith(entityType.getSimpleName() + "s.xml");+ }++ });+ for (File file: dataFiles) {+ Transaction transaction = session.beginTransaction();+ try {+ logger.info("Importing from data file '" + file.getName() + "'...");+ VersionedDocument dom = VersionedDocument.fromFile(file);++ for (Element element: dom.getRootElement().elements()) {+ element.detach();+ AbstractEntity entity = (AbstractEntity) new VersionedDocument(DocumentHelper.createDocument(element)).toBean();+ session.replicate(entity, ReplicationMode.EXCEPTION);+ }+ session.flush();+ session.clear();+ transaction.commit();+ } catch (Exception e) {+ transaction.rollback();+ throw ExceptionUtils.unchecked(e);+ }+ }+ }+ }++ protected void validateData(Metadata metadata, File dataDir) {+ List<Class<?>> entityTypes = getEntityTypes(sessionFactory);+ Collections.reverse(entityTypes);+ for (Class<?> entityType: entityTypes) {+ File[] dataFiles = dataDir.listFiles(new FilenameFilter() {++ @Override+ public boolean accept(File dir, String name) {+ return name.startsWith(entityType.getSimpleName() + "s.xml");+ }++ });+ for (File file: dataFiles) {+ try {+ logger.info("Validating data file '" + file.getName() + "'...");+ VersionedDocument dom = VersionedDocument.fromFile(file);++ for (Element element: dom.getRootElement().elements()) {+ element.detach();+ AbstractEntity entity = (AbstractEntity) new VersionedDocument(DocumentHelper.createDocument(element)).toBean();+ validator.validate(entity);+ }+ } catch (Exception e) {+ throw ExceptionUtils.unchecked(e);+ }+ }+ }+ }++ protected void applyConstraints(Metadata metadata) {+ File tempFile = null;+ try {+ tempFile = File.createTempFile("schema", ".sql");+ new SchemaExport().setOutputFile(tempFile.getAbsolutePath())+ .setFormat(false).createOnly(EnumSet.of(TargetType.SCRIPT), metadata);+ List<String> sqls = new ArrayList<>();+ for (String sql: FileUtils.readLines(tempFile, Charset.defaultCharset())) {+ if (isApplyingConstraints(sql)) {+ sqls.add(sql);+ }+ }+ execute(sqls, true);+ } catch (IOException e) {+ throw new RuntimeException(e);+ } finally {+ if (tempFile != null)+ tempFile.delete();+ }+ }++ protected void createTables(Metadata metadata) {+ File tempFile = null;+ try {+ tempFile = File.createTempFile("schema", ".sql");+ new SchemaExport().setOutputFile(tempFile.getAbsolutePath())+ .setFormat(false).createOnly(EnumSet.of(TargetType.SCRIPT), metadata);+ List<String> sqls = new ArrayList<>();+ for (String sql: FileUtils.readLines(tempFile, Charset.defaultCharset())) {+ if (shouldInclude(sql) && !isApplyingConstraints(sql))+ sqls.add(sql);+ }+ execute(sqls, true);+ } catch (IOException e) {+ throw new RuntimeException(e);+ } finally {+ if (tempFile != null)+ FileUtils.deleteFile(tempFile);+ }+ }++ protected void dropConstraints(Metadata metadata) {+ File tempFile = null;+ try {+ tempFile = File.createTempFile("schema", ".sql");+ new SchemaExport().setOutputFile(tempFile.getAbsolutePath())+ .setFormat(false).drop(EnumSet.of(TargetType.SCRIPT), metadata);+ List<String> sqls = new ArrayList<>();+ for (String sql: FileUtils.readLines(tempFile, Charset.defaultCharset())) {+ if (isDroppingConstraints(sql))+ sqls.add(sql);+ }+ execute(sqls, false);+ } catch (IOException e) {+ throw new RuntimeException(e);+ } finally {+ if (tempFile != null)+ tempFile.delete();+ }+ }++ protected void cleanDatabase(Metadata metadata) {+ File tempFile = null;+ try {+ tempFile = File.createTempFile("schema", ".sql");+ new SchemaExport().setOutputFile(tempFile.getAbsolutePath())+ .setFormat(false).drop(EnumSet.of(TargetType.SCRIPT), metadata);+ List<String> sqls = new ArrayList<>();+ for (String sql: FileUtils.readLines(tempFile, Charset.defaultCharset())) {+ sqls.add(sql);+ }+ execute(sqls, false);+ } catch (IOException e) {+ throw new RuntimeException(e);+ } finally {+ if (tempFile != null)+ tempFile.delete();+ }+ }++ private boolean isApplyingConstraints(String sql) {+ return sql.toLowerCase().contains(" foreign key ");+ }++ private boolean isDroppingConstraints(String sql) {+ return sql.toLowerCase().contains(" drop constraint ");+ }++ private boolean shouldInclude(String sql) {+ // some databases will create index on foreign keys automatically, so+ // we skip creating indexes for those+ if (!sql.toLowerCase().startsWith("create index") || !sql.toLowerCase().endsWith("_id)")) {+ return true;+ } else {+ String dialect = getDialect().toLowerCase();+ return !dialect.contains("mysql") && !dialect.contains("hsql");+ }+ }++ @Override+ public SessionFactory getSessionFactory() {+ return sessionFactory;+ }++}diff --git a/server-product/system/conf/logback.xml b/server-product/system/conf/logback.xmlindex 1c386f58bd..99b98954ff 100644--- a/server-product/system/conf/logback.xml+++ b/server-product/system/conf/logback.xml@@ -40,6 +40,7 @@<logger name="com.zaxxer.hikari.HikariDataSource" level="ERROR"/><logger name="com.zaxxer.hikari.util.DriverDataSource" level="ERROR"/><logger name="org.apache.shiro.mgt" level="ERROR"/>+ <logger name="org.apache.wicket.request.resource.ResourceReferenceRegistry" level="ERROR"/><logger name="io.onedev" level="INFO"/><root level="WARN">