diff --git a/api/src/org/labkey/api/dataiterator/DataClassDataIteratorTransformer.java b/api/src/org/labkey/api/dataiterator/DataClassDataIteratorTransformer.java new file mode 100644 index 00000000000..2c869de2560 --- /dev/null +++ b/api/src/org/labkey/api/dataiterator/DataClassDataIteratorTransformer.java @@ -0,0 +1,31 @@ +package org.labkey.api.dataiterator; + +import org.jetbrains.annotations.NotNull; + +import java.util.Map; + +/** + * Extension point for DataClass-specific transformations in the pre-trigger DataIterator pipeline. + * Registered per DataClass name via {@link org.labkey.api.exp.api.ExperimentService#registerDataClassDataIteratorTransformer}. + * A fresh instance is created for each import operation since implementations may be stateful + * between {@link #prepareTranslator} and {@link #wrapDataIterator}. + */ +public interface DataClassDataIteratorTransformer +{ + /** + * Called during SimpleTranslator setup to inspect input columns and add placeholder columns + * (e.g., via {@link SimpleTranslator#addNullColumn}) that will be populated by the wrapping DataIterator. + * + * @return true if the transformer is active and {@link #wrapDataIterator} should be called + */ + boolean prepareTranslator(@NotNull SimpleTranslator step0, + @NotNull Map inputColumnNameMap, + @NotNull DataIteratorContext context); + + /** + * Wraps the DataIterator to apply DataClass-specific transformations. + * Called after the pre-trigger pipeline is assembled, only if {@link #prepareTranslator} returned true. + */ + @NotNull + DataIterator wrapDataIterator(@NotNull DataIterator input, @NotNull DataIteratorContext context); +} diff --git a/api/src/org/labkey/api/exp/api/ExperimentService.java b/api/src/org/labkey/api/exp/api/ExperimentService.java index 0a73f2544f0..ce8efc452e8 100644 --- a/api/src/org/labkey/api/exp/api/ExperimentService.java +++ b/api/src/org/labkey/api/exp/api/ExperimentService.java @@ -32,6 +32,7 @@ import org.labkey.api.data.RemapCache; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.TableInfo; +import org.labkey.api.dataiterator.DataClassDataIteratorTransformer; import org.labkey.api.exp.ExperimentDataHandler; import org.labkey.api.exp.ExperimentException; import org.labkey.api.exp.ExperimentProtocolHandler; @@ -74,6 +75,7 @@ import org.labkey.api.pipeline.RecordedActionSet; import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.FilteredTable; +import org.labkey.api.query.QueryUpdateService; import org.labkey.api.query.QueryKey; import org.labkey.api.query.QueryViewProvider; import org.labkey.api.query.UserSchema; @@ -101,6 +103,8 @@ import java.util.Set; import java.util.concurrent.locks.Lock; import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.function.UnaryOperator; import static org.labkey.api.exp.api.ExpDataClass.NEW_DATA_CLASS_ALIAS_VALUE; import static org.labkey.api.exp.api.SampleTypeService.NEW_SAMPLE_TYPE_ALIAS_VALUE; @@ -657,6 +661,21 @@ static void validateParentAlias(Map aliasMap, Set reserv ExpDataClassDataTable createDataClassDataTable(String name, UserSchema schema, ContainerFilter cf, @NotNull ExpDataClass dataClass); + /** + * Registers a factory that creates a {@link DataClassDataIteratorTransformer} + * for the specified DataClass name. The transformer is applied in the pre-trigger DataIterator pipeline, + * allowing modules to add computed columns (e.g., transforming flat columns into JSON) that work + * uniformly for file imports, API imports, folder imports, and background pipeline jobs. + * A fresh instance is created per import via the factory since transformers may be stateful. + */ + void registerDataClassDataIteratorTransformer(String dataClassName, @NotNull Supplier factory); + + /** + * Returns a fresh {@link DataClassDataIteratorTransformer} for the given + * DataClass name, or {@code null} if none is registered. + */ + @Nullable DataClassDataIteratorTransformer getDataClassDataIteratorTransformer(String dataClassName); + ExpProtocolTable createProtocolTable(String name, UserSchema schema, ContainerFilter cf); ExpExperimentTable createExperimentTable(String name, UserSchema schema, ContainerFilter cf); diff --git a/api/src/org/labkey/api/pipeline/AppPipelineJobNotificationProvider.java b/api/src/org/labkey/api/pipeline/AppPipelineJobNotificationProvider.java index 2829f43f2d1..d236b975d9d 100644 --- a/api/src/org/labkey/api/pipeline/AppPipelineJobNotificationProvider.java +++ b/api/src/org/labkey/api/pipeline/AppPipelineJobNotificationProvider.java @@ -31,6 +31,7 @@ abstract public class AppPipelineJobNotificationProvider implements PipelineJobN public enum ImportType { samples, sources, + registry, assays; public static ImportType getImportType(PipelineJob job) @@ -41,7 +42,11 @@ public static ImportType getImportType(PipelineJob job) if (schemaName.equalsIgnoreCase("samples")) return samples; else if (schemaName.equalsIgnoreCase("exp.data")) + { + if ("Biologics".equalsIgnoreCase(queryImportPipelineJob.getImportContextBuilder().getJobNotificationProvider())) + return registry; return sources; + } else if (schemaName.equalsIgnoreCase("exp")) { String queryName = queryImportPipelineJob.getImportContextBuilder().getQueryName(); @@ -125,29 +130,8 @@ public URLHelper getPipelineStatusHref(PipelineJob job) if (importType == null) return null; - String urlFragment = "/" + importType.name(); - - ActionURL appURL = getAppURL(job.getContainer()); - - String type = queryImportPipelineJob.getImportContextBuilder().getQueryName(); - if (!"materials".equalsIgnoreCase(type) || !"exp".equalsIgnoreCase(queryImportPipelineJob.getImportContextBuilder().getSchemaName())) - urlFragment += "/" + type + "?"; - else - urlFragment += "?"; - - String and = ""; - - if (queryImportPipelineJob.getTransactionAuditId() > 0) - { - urlFragment = urlFragment + "transactionAuditId=" + queryImportPipelineJob.getTransactionAuditId(); - and = "&"; - } - - String filename = queryImportPipelineJob.getImportContextBuilder().getPrimaryFile().getName(); - if (!StringUtils.isEmpty(filename)) - urlFragment = urlFragment + and + "importFile=" + PageFlowUtil.encode(filename); - - return appURL.setFragment(urlFragment); + String urlFragment = buildQueryImportUrlFragment(queryImportPipelineJob, importType, queryImportPipelineJob.getAdditionalJobResponseInfo()); + return getAppURL(job.getContainer()).setFragment(urlFragment); } return null; @@ -240,36 +224,58 @@ else if (job instanceof AssayUploadPipelineJob assayJob) return null; } - private String getJobSuccessUrl(PipelineJob job, @NotNull ImportType importType, @Nullable Map info) + private String buildQueryImportUrlFragment(QueryImportPipelineJob queryImportPipelineJob, ImportType importType, @Nullable Map info) { - ActionURL appURL = getAppURL(job.getContainer()); String urlFragment = "/" + importType.name(); - if (job instanceof QueryImportPipelineJob queryImportPipelineJob) + + Boolean isCrossType = queryImportPipelineJob.getImportContextBuilder().getOptionParamsMap().get(AbstractQueryImportAction.Params.crossTypeImport); + if (Boolean.TRUE.equals(isCrossType)) + urlFragment = "/crossType/" + importType.name() + "?"; + else if (info != null && info.containsKey("viewJobDataUrl")) + { + urlFragment = info.get("viewJobDataUrl").toString(); + if (!urlFragment.endsWith("?") && !urlFragment.endsWith("&")) + urlFragment += "?"; + } + else { - Boolean isCrossType = queryImportPipelineJob.getImportContextBuilder().getOptionParamsMap().get(AbstractQueryImportAction.Params.crossTypeImport); - if (isCrossType) - urlFragment = "/crossType/" + importType.name() + "?"; + String type = queryImportPipelineJob.getImportContextBuilder().getQueryName(); + String schemaName = queryImportPipelineJob.getImportContextBuilder().getSchemaName(); + if ("materials".equalsIgnoreCase(type) && "exp".equalsIgnoreCase(schemaName)) + urlFragment += "?"; else - { - String type = queryImportPipelineJob.getImportContextBuilder().getQueryName(); urlFragment += "/" + PageFlowUtil.encode(type) + "?"; - } + } - String and = ""; - if (info != null) + String and = ""; + if (info != null) + { + Long transactionAuditId = asLong(info.get("transactionAuditId")); + if (transactionAuditId != null && transactionAuditId > 0) { - Long transactionAuditId = (Long) info.get("transactionAuditId"); - urlFragment = urlFragment + "transactionAuditId=" + transactionAuditId ; + urlFragment = urlFragment + "transactionAuditId=" + transactionAuditId; and = "&"; } + } - String filename = queryImportPipelineJob.getImportContextBuilder().getPrimaryFile().getName(); - if (!StringUtils.isEmpty(filename)) - urlFragment = urlFragment + and + "importFile=" + PageFlowUtil.encode(filename); + String filename = queryImportPipelineJob.getImportContextBuilder().getPrimaryFile().getName(); + if (!StringUtils.isEmpty(filename)) + urlFragment = urlFragment + and + "importFile=" + PageFlowUtil.encode(filename); + return urlFragment; + } + + private String getJobSuccessUrl(PipelineJob job, @NotNull ImportType importType, @Nullable Map info) + { + ActionURL appURL = getAppURL(job.getContainer()); + String urlFragment; + if (job instanceof QueryImportPipelineJob queryImportPipelineJob) + { + urlFragment = buildQueryImportUrlFragment(queryImportPipelineJob, importType, info); } else if (job instanceof AssayUploadPipelineJob) { + urlFragment = "/" + importType.name(); if (info != null) { String provider = (String) info.get("provider"); @@ -278,6 +284,10 @@ else if (job instanceof AssayUploadPipelineJob) urlFragment = urlFragment + "/" + PageFlowUtil.encode(provider) + "/" + PageFlowUtil.encode(assayName) + "/runs/" + runId; } } + else + { + urlFragment = "/" + importType.name(); + } return appURL.setFragment(urlFragment).getLocalURIString(); } diff --git a/api/src/org/labkey/api/query/AbstractQueryImportAction.java b/api/src/org/labkey/api/query/AbstractQueryImportAction.java index 0480c0fb7be..9e489a5165a 100644 --- a/api/src/org/labkey/api/query/AbstractQueryImportAction.java +++ b/api/src/org/labkey/api/query/AbstractQueryImportAction.java @@ -582,7 +582,7 @@ else if (!dataFileDir.exists()) multipartfile.transferTo(dataFile.toNioPathForWrite()); if (_useAsync) { - if (!isBackgroundImportSupported()) + if (!isBackgroundImportSupported(dataFile.getName())) throw new RuntimeException("Importing in background currently is not supported for this table"); ViewBackgroundInfo info = new ViewBackgroundInfo(getContainer(), getUser(), new ActionURL()); @@ -757,7 +757,7 @@ protected String getQueryImportJobNotificationProviderName() return null; } - protected boolean isBackgroundImportSupported() + protected boolean isBackgroundImportSupported(@NotNull String fileName) { return false; } diff --git a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java index 27d7080a0ab..19453164ec4 100644 --- a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java +++ b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java @@ -399,6 +399,11 @@ protected int _importRowsUsingDIB(User user, Container container, DataIteratorBu if (extraScriptContext != null) { context.setDataSource((String) extraScriptContext.get(DataIteratorUtil.DATA_SOURCE)); + if (extraScriptContext.containsKey(AbstractQueryImportAction.Params.useTransactionAuditCache.name())) + { + boolean useTransactionAuditCache = Boolean.TRUE.equals(extraScriptContext.get(AbstractQueryImportAction.Params.useTransactionAuditCache.name())); + context.setUseTransactionAuditCache(useTransactionAuditCache); + } } preImportDIBValidation(in, null); diff --git a/api/src/org/labkey/api/query/QueryImportPipelineJob.java b/api/src/org/labkey/api/query/QueryImportPipelineJob.java index cc236d69a4f..5daf9039a09 100644 --- a/api/src/org/labkey/api/query/QueryImportPipelineJob.java +++ b/api/src/org/labkey/api/query/QueryImportPipelineJob.java @@ -36,6 +36,8 @@ public class QueryImportPipelineJob extends PipelineJob private long _transactionAuditId; + private Map _additionalJobResponseInfo; + protected QueryImportPipelineJob() {} @@ -279,7 +281,7 @@ public URLHelper getStatusHref() } url = target.getGridURL(getContainer()); - if (_transactionAuditId > 0) + if (url != null && _transactionAuditId > 0) url.addParameter("transactionAuditId", String.valueOf(_transactionAuditId)); return url; @@ -343,9 +345,6 @@ public void run() if (auditEvent != null) _transactionAuditId = auditEvent.getRowId(); - setStatus(TaskStatus.complete); - getLogger().info("Done importing {}. {} row(s) imported.", getDescription(), importedCount); - if (notificationProvider != null) { Map results = new HashMap<>(); @@ -359,8 +358,15 @@ public void run() if (!diContext.getResponseInfo().isEmpty()) results.putAll(diContext.getResponseInfo()); - notificationProvider.onJobSuccess(this, results); + + _additionalJobResponseInfo = results; } + + setStatus(TaskStatus.complete); + getLogger().info("Done importing {}. {} row(s) imported.", getDescription(), importedCount); + + if (notificationProvider != null) + notificationProvider.onJobSuccess(this, getAdditionalJobResponseInfo()); } catch (QueryImportJobCancelledException e) { @@ -415,4 +421,9 @@ public long getTransactionAuditId() return _transactionAuditId; } + public Map getAdditionalJobResponseInfo() + { + return _additionalJobResponseInfo; + } + } diff --git a/audit/src/org/labkey/audit/AuditController.java b/audit/src/org/labkey/audit/AuditController.java index de464e6c5d5..0786e23dd51 100644 --- a/audit/src/org/labkey/audit/AuditController.java +++ b/audit/src/org/labkey/audit/AuditController.java @@ -69,7 +69,6 @@ import java.util.Collections; import java.util.Date; import java.util.GregorianCalendar; -import java.util.List; import java.util.Map; import static org.labkey.api.data.ContainerManager.REQUIRE_USER_COMMENTS_PROPERTY_NAME; @@ -382,17 +381,18 @@ public void validateForm(AuditTransactionForm form, Errors errors) @Override public Object execute(AuditTransactionForm form, BindException errors) { - List rowIds; + AuditLogImpl.TransactionRowIds results; User elevatedUser = ElevatedUser.ensureCanSeeAuditLogRole(getContainer(), getUser()); ContainerFilter cf = ContainerFilter.getContainerFilterByName(form.getContainerFilter(), getContainer(), elevatedUser); if (form.isSampleType()) - rowIds = AuditLogImpl.get().getTransactionSampleIds(form.getTransactionAuditId(), elevatedUser, getContainer(), cf); + results = AuditLogImpl.get().getTransactionSampleIds(form.getTransactionAuditId(), elevatedUser, getContainer(), cf); else - rowIds = AuditLogImpl.get().getTransactionSourceIds(form.getTransactionAuditId(), elevatedUser, getContainer(), cf); + results = AuditLogImpl.get().getTransactionSourceIds(form.getTransactionAuditId(), elevatedUser, getContainer(), cf); ApiSimpleResponse response = new ApiSimpleResponse(); response.put("success", true); - response.put("rowIds", rowIds); + response.put("rowIds", results.rowIds()); + response.put("dataTypeRowCounts", results.dataTypeRowCounts()); return response; } diff --git a/audit/src/org/labkey/audit/AuditLogImpl.java b/audit/src/org/labkey/audit/AuditLogImpl.java index 3adc44bb5fe..bf68aa085fd 100644 --- a/audit/src/org/labkey/audit/AuditLogImpl.java +++ b/audit/src/org/labkey/audit/AuditLogImpl.java @@ -55,6 +55,7 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -247,63 +248,63 @@ public ActionURL getAuditUrl() return new ActionURL(AuditController.ShowAuditLogAction.class, ContainerManager.getRoot()); } - public List getTransactionSampleIds(long transactionAuditId, User user, Container container, @Nullable ContainerFilter containerFilter) + public record TransactionRowIds(List rowIds, Map dataTypeRowCounts) {} + + public TransactionRowIds getTransactionSampleIds(long transactionAuditId, User user, Container container, @Nullable ContainerFilter containerFilter) { List transactionEvents = TRANSACTION_EVENT_CACHE.get(transactionAuditId).second; - if (!transactionEvents.isEmpty()) + List events; + if (transactionEvents.isEmpty()) { - List ids = new ArrayList<>(); - transactionEvents.forEach(event -> { - if (event instanceof SampleTimelineAuditEvent stEvent) - ids.add(stEvent.getSampleId()); - }); - return ids; + SimpleFilter filter = new SimpleFilter(); + filter.addCondition(FieldKey.fromParts("TransactionID"), transactionAuditId); + events = AuditLogService.get().getAuditEvents(container, user, SampleTimelineAuditEvent.EVENT_TYPE, filter, null, containerFilter); } - - SimpleFilter filter = new SimpleFilter(); - filter.addCondition(FieldKey.fromParts("TransactionID"), transactionAuditId); - - List events = AuditLogService.get().getAuditEvents(container, user, SampleTimelineAuditEvent.EVENT_TYPE, filter, null, containerFilter); - return events.stream().map(SampleTimelineAuditEvent::getSampleId).collect(Collectors.toList()); + else + { + events = transactionEvents.stream() + .filter(SampleTimelineAuditEvent.class::isInstance) + .map(SampleTimelineAuditEvent.class::cast) + .toList(); + } + Map dataTypeRowCounts = new HashMap<>(); + List sampleIds = new ArrayList<>(); + events.forEach(event -> { + dataTypeRowCounts.merge(event.getSampleTypeId(), 1L, Long::sum); + sampleIds.add(event.getSampleId()); + }); + return new TransactionRowIds(sampleIds, dataTypeRowCounts); } - public List getTransactionSourceIds(long transactionAuditId, User user, Container container, @Nullable ContainerFilter containerFilter) + public TransactionRowIds getTransactionSourceIds(long transactionAuditId, User user, Container container, @Nullable ContainerFilter containerFilter) { List lsids = new ArrayList<>(); List sourceIds = new ArrayList<>(); + Map dataTypeRowCounts = new HashMap<>(); List transactionEvents = TRANSACTION_EVENT_CACHE.get(transactionAuditId).second; - if (!transactionEvents.isEmpty()) - { - transactionEvents.forEach(event -> { - if (event instanceof DetailedAuditTypeEvent detailedEvent) - { - if (detailedEvent.getNewRecordMap() != null) - { - Map newRecord = new CaseInsensitiveHashMap<>(AbstractAuditTypeProvider.decodeFromDataMap(detailedEvent.getNewRecordMap())); - if (newRecord.containsKey("RowId") && !StringUtils.isEmpty(newRecord.get("RowId"))) - sourceIds.add(Long.valueOf(newRecord.get("RowId"))); - else if (newRecord.containsKey("LSID") && !StringUtils.isEmpty(newRecord.get("LSID"))) - lsids.add(newRecord.get("LSID")); - } - } - }); - } - else - { - List events = QueryService.get().getQueryUpdateAuditRecords(user, container, transactionAuditId, containerFilter); + List detailedEvents = transactionEvents.isEmpty() + ? QueryService.get().getQueryUpdateAuditRecords(user, container, transactionAuditId, containerFilter) + : transactionEvents.stream() + .filter(DetailedAuditTypeEvent.class::isInstance) + .map(DetailedAuditTypeEvent.class::cast) + .toList(); + + detailedEvents.forEach(event -> { + if (event.getNewRecordMap() != null) + { + Map newRecord = new CaseInsensitiveHashMap<>(AbstractAuditTypeProvider.decodeFromDataMap(event.getNewRecordMap())); + if (newRecord.containsKey("RowId") && !StringUtils.isEmpty(newRecord.get("RowId"))) + sourceIds.add(Long.valueOf(newRecord.get("RowId"))); + else if (newRecord.containsKey("LSID") && !StringUtils.isEmpty(newRecord.get("LSID"))) + lsids.add(newRecord.get("LSID")); - events.forEach((event) -> { - if (event.getNewRecordMap() != null) + if (newRecord.containsKey("ClassId") && !StringUtils.isEmpty(newRecord.get("ClassId"))) { - Map newRecord = new CaseInsensitiveHashMap<>(AbstractAuditTypeProvider.decodeFromDataMap(event.getNewRecordMap())); - if (newRecord.containsKey("RowId") && !StringUtils.isEmpty(newRecord.get("RowId"))) - sourceIds.add(Long.valueOf(newRecord.get("RowId"))); - else if (newRecord.containsKey("LSID") && !StringUtils.isEmpty(newRecord.get("LSID"))) - lsids.add(newRecord.get("LSID")); - + Long classId = Long.valueOf(newRecord.get("ClassId")); + dataTypeRowCounts.merge(classId, 1L, Long::sum); } - }); - } + } + }); if (!lsids.isEmpty()) { SimpleFilter filter = SimpleFilter.createContainerFilter(container); @@ -311,6 +312,6 @@ else if (newRecord.containsKey("LSID") && !StringUtils.isEmpty(newRecord.get("LS TableSelector selector = new TableSelector(ExperimentService.get().getTinfoData(), Collections.singleton("RowId"), filter, null); sourceIds.addAll(selector.getArrayList(Long.class)); } - return sourceIds; + return new TransactionRowIds(sourceIds, dataTypeRowCounts); } } diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index cafc1fc8f07..f5f81746a2c 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -96,6 +96,7 @@ import org.labkey.api.exp.query.SamplesSchema; import org.labkey.api.qc.DataState; import org.labkey.api.qc.SampleStatusService; +import org.labkey.api.query.AbstractQueryImportAction; import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.FieldKey; import org.labkey.api.query.FileColumnValueMapper; @@ -2379,7 +2380,10 @@ public DataIterator getDataIterator(DataIteratorContext context) // useTransactionAuditCache already set for import and merge in AbstractQueryImportAction.createDataIteratorContext if (context.getInsertOption() == QueryUpdateService.InsertOption.INSERT) - context.setUseTransactionAuditCache(true); + { + if (Boolean.FALSE != context.getConfigParameter(AbstractQueryImportAction.Params.useTransactionAuditCache)) + context.setUseTransactionAuditCache(true); + } // add FileLink DataIterator if any input columns are of type FILE_LINK if (null != _fileLinkDirectory) diff --git a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java index 8f3bd242924..2c47efa7184 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java @@ -52,6 +52,7 @@ import org.labkey.api.data.dialect.SqlDialect; import org.labkey.api.dataiterator.AttachmentDataIterator; import org.labkey.api.dataiterator.CachingDataIterator; +import org.labkey.api.dataiterator.DataClassDataIteratorTransformer; import org.labkey.api.dataiterator.CoerceDataIterator; import org.labkey.api.dataiterator.DataClassUpdateAddColumnsDataIterator; import org.labkey.api.dataiterator.DataIterator; @@ -85,6 +86,7 @@ import org.labkey.api.exp.query.ExpDataTable; import org.labkey.api.exp.query.ExpSchema; import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.query.AbstractQueryImportAction; import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.DefaultQueryUpdateService; import org.labkey.api.query.DetailsURL; @@ -143,6 +145,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Supplier; +import java.util.function.UnaryOperator; import java.util.stream.Collectors; import static org.labkey.api.dataiterator.DataIteratorUtil.DUPLICATE_COLUMN_IN_DATA_ERROR; @@ -1119,9 +1122,20 @@ else if (Column.ClassId.name().equalsIgnoreCase(name)) final int batchSize = _context.getInsertOption().batch ? BATCH_SIZE : 1; step0.addSequenceColumn(genIdCol, _dataClass.getContainer(), ExpDataClassImpl.SEQUENCE_PREFIX, _dataClass.getRowId(), batchSize, _dataClass.getMinGenId()); + // Apply registered DataClass-specific DataIterator transformer (e.g., Molecule Component-N/X → components JSON) + DataIteratorBuilder step1 = step0; + DataClassDataIteratorTransformer transformer = ExperimentService.get().getDataClassDataIteratorTransformer(_dataClass.getName()); + if (transformer != null) + { + if (transformer.prepareTranslator(step0, columnNameMap, context)) + step1 = LoggingDataIterator.wrap(transformer.wrapDataIterator(step0, context)); + if (context.getErrors().hasErrors()) + return null; + } + // Table Counters ExpDataClassDataTableImpl queryTable = ExpDataClassDataTableImpl.this; - var counterDIB = ExpDataIterators.CounterDataIteratorBuilder.create(step0, _dataClass.getContainer(), queryTable, ExpDataClassImpl.SEQUENCE_PREFIX, _dataClass.getRowId()); + var counterDIB = ExpDataIterators.CounterDataIteratorBuilder.create(step1, _dataClass.getContainer(), queryTable, ExpDataClassImpl.SEQUENCE_PREFIX, _dataClass.getRowId()); DataIterator di; // Generate names @@ -1210,7 +1224,7 @@ protected DataIteratorBuilder preTriggerDataIterator(DataIteratorBuilder in, Dat @Override public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) { - return _importRowsUsingDIB(user, container, rows, null, getDataIteratorContext(errors, InsertOption.INSERT, configParameters), extraScriptContext); + return _importRowsUsingDIB(user, container, rows, null, getDataIteratorContext(errors, InsertOption.IMPORT, configParameters), extraScriptContext); } @Override @@ -1299,7 +1313,11 @@ private Map> getDataClassObjectsForMoveRows(Container t @Override public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) { - return super._insertRowsUsingDIB(user, container, rows, getDataIteratorContext(errors, InsertOption.INSERT, configParameters), extraScriptContext); + InsertOption insertOption = InsertOption.INSERT; + // allow caller to use InsertOption.IMPORT to useImportAliases + if (configParameters != null && configParameters.get(AbstractQueryImportAction.Params.insertOption) instanceof InsertOption option) + insertOption = option; + return super._insertRowsUsingDIB(user, container, rows, getDataIteratorContext(errors, insertOption, configParameters), extraScriptContext); } @Override diff --git a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java index f79db389506..bf51266dc3f 100644 --- a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java @@ -94,6 +94,7 @@ import org.labkey.api.data.TableSelector; import org.labkey.api.data.TempTableTracker; import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.dataiterator.DataClassDataIteratorTransformer; import org.labkey.api.dataiterator.DataIteratorBuilder; import org.labkey.api.defaults.DefaultValueService; import org.labkey.api.exp.AbstractParameter; @@ -284,11 +285,13 @@ import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Predicate; import java.util.function.Supplier; +import java.util.function.UnaryOperator; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -353,6 +356,7 @@ public class ExperimentServiceImpl implements ExperimentService, ObjectReference private final Map _dataTypes = new HashMap<>(); private final Map _protocolImplementations = new HashMap<>(); private final Map _protocolInputCriteriaFactories = new HashMap<>(); + private final Map> _dataClassDataIteratorTransformers = new ConcurrentHashMap<>(); private final Set _protocolHandlers = new HashSet<>(); private final List _objectReferencers = new ArrayList<>(); private final List _columnExporters = new ArrayList<>(); @@ -1590,6 +1594,19 @@ public ExpDataClassDataTable createDataClassDataTable(String name, UserSchema sc return new ExpDataClassDataTableImpl(name, schema, cf, (ExpDataClassImpl) dataClass); } + @Override + public void registerDataClassDataIteratorTransformer(String dataClassName, @NotNull Supplier factory) + { + _dataClassDataIteratorTransformers.put(dataClassName.toLowerCase(), factory); + } + + @Override + public @Nullable DataClassDataIteratorTransformer getDataClassDataIteratorTransformer(String dataClassName) + { + Supplier factory = _dataClassDataIteratorTransformers.get(dataClassName.toLowerCase()); + return factory != null ? factory.get() : null; + } + @Override public ExpMaterialInputTable createMaterialInputTable(String name, ExpSchema schema, ContainerFilter cf) { diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 0c9ab919886..92510061fc5 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -84,6 +84,7 @@ import org.labkey.api.ontology.Unit; import org.labkey.api.qc.DataState; import org.labkey.api.qc.SampleStatusService; +import org.labkey.api.query.AbstractQueryImportAction; import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.DefaultQueryUpdateService; import org.labkey.api.query.FieldKey; @@ -241,7 +242,7 @@ public int importRows(User user, Container container, DataIteratorBuilder rows, ArrayList> outputRows = new ArrayList<>(); Map finalConfigParameters = configParameters == null ? new HashMap<>() : configParameters; finalConfigParameters.put(ExperimentService.QueryOptions.GetSampleRecomputeCol, true); - int ret = _importRowsUsingDIB(user, container, rows, outputRows, getDataIteratorContext(errors, InsertOption.INSERT, finalConfigParameters), extraScriptContext); + int ret = _importRowsUsingDIB(user, container, rows, outputRows, getDataIteratorContext(errors, InsertOption.IMPORT, finalConfigParameters), extraScriptContext); if (ret > 0 && !errors.hasErrors()) { onSamplesChanged(outputRows, configParameters, container, insert); @@ -487,9 +488,18 @@ public List> insertRows(User user, Container container, List assert _sampleType != null : "SampleType required for insert/update, but not required for read/delete"; // insertRows with lineage is pretty good at deadlocking against itself, so use retry loop + InsertOption insertOption; + // allow caller to use InsertOption.IMPORT to useImportAliases + if (configParameters != null && configParameters.get(AbstractQueryImportAction.Params.insertOption) instanceof InsertOption option) + insertOption = option; + else + { + insertOption = InsertOption.INSERT; + } + DbScope scope = getSchema().getDbSchema().getScope(); List> results = scope.executeWithRetry(transaction -> - super._insertRowsUsingDIB(user, container, rows, getDataIteratorContext(errors, InsertOption.INSERT, configParameters), extraScriptContext)); + super._insertRowsUsingDIB(user, container, rows, getDataIteratorContext(errors, insertOption, configParameters), extraScriptContext)); if (results != null && !results.isEmpty() && !errors.hasErrors()) { diff --git a/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java b/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java index 7ea9f73f761..27f2e4eaf57 100644 --- a/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java +++ b/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java @@ -4561,7 +4561,7 @@ protected String getQueryImportJobNotificationProviderName() } @Override - protected boolean isBackgroundImportSupported() + protected boolean isBackgroundImportSupported(@NotNull String fileName) { return true; }