From 675ec6eaef4074b86380ad868cf4ff6ed9f0ce77 Mon Sep 17 00:00:00 2001 From: felmer <franz-josef.elmer@id.ethz.ch> Date: Tue, 18 Sep 2018 10:55:14 +0200 Subject: [PATCH] SSDM-7171: column 'number of entities' added if property count-all-entities = true. New unit test added. --- .../server/task/UsageReportingTask.java | 107 ++++++++++++-- .../server/task/UsageReportingTaskTest.java | 132 ++++++++++++++---- 2 files changed, 200 insertions(+), 39 deletions(-) diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/task/UsageReportingTask.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/task/UsageReportingTask.java index 70e00c146cd..1750a9a3d6c 100644 --- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/task/UsageReportingTask.java +++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/task/UsageReportingTask.java @@ -40,6 +40,7 @@ import org.apache.commons.lang.time.DateUtils; import ch.systemsx.cisd.common.mail.EMailAddress; import ch.systemsx.cisd.common.mail.IMailClient; import ch.systemsx.cisd.common.maintenance.MaintenanceTaskParameters; +import ch.systemsx.cisd.common.properties.PropertyUtils; import ch.systemsx.cisd.common.time.DateTimeUtils; import ch.systemsx.cisd.openbis.generic.server.CommonServiceProvider; import ch.systemsx.cisd.openbis.generic.server.util.PluginUtils; @@ -51,6 +52,13 @@ import ch.systemsx.cisd.openbis.generic.server.util.PluginUtils; */ public class UsageReportingTask extends AbstractMaintenanceTask { + public static interface IUsageInfoHandler + { + public void handleUsageInfo(GroupInfo groupInfo, String user, UsageInfo usageInfo, boolean groupAction); + + public void handleGroupUsageInfo(GroupInfo groupInfo, String user, UsageInfo usageInfo); + } + enum UserReportingType { NONE(), OUTSIDE_GROUP_ONLY() @@ -80,9 +88,12 @@ public class UsageReportingTask extends AbstractMaintenanceTask static final String USER_REPORTING_KEY = "user-reporting-type"; + static final String COUNT_ALL_ENTITIES_KEY = "count-all-entities"; + static final String DELIM = "\t"; private static final String DATE_FORMAT = "yyyy-MM-dd"; + private static final String TIME_STAMP_FORMAT = "yyyy-MM-dd HH:mm:ss"; private PeriodType periodType; @@ -91,6 +102,8 @@ public class UsageReportingTask extends AbstractMaintenanceTask private UserReportingType userReportingType; + private boolean countAllEntities; + public UsageReportingTask() { super(false); @@ -103,19 +116,26 @@ public class UsageReportingTask extends AbstractMaintenanceTask periodType = PeriodType.getBestType(interval); eMailAddresses = PluginUtils.getEMailAddresses(properties, ","); userReportingType = UserReportingType.valueOf(properties.getProperty(USER_REPORTING_KEY, UserReportingType.ALL.name())); + countAllEntities = PropertyUtils.getBoolean(properties, COUNT_ALL_ENTITIES_KEY, false); } @Override public void execute() { List<String> groups = getGroups(); - Period period = periodType.getPeriod(getActualTimeStamp()); + Date actualTimeStamp = getActualTimeStamp(); + Period period = periodType.getPeriod(actualTimeStamp); SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT); String fromDateString = dateFormat.format(period.getFrom()); String untilDateString = dateFormat.format(period.getUntil()); operationLog.info("Gather usage information for the period from " + fromDateString + " until " + untilDateString); UsageAndGroupsInfo usageAndGroupsInfo = gatherUsageAndGroups(groups, period); - String report = createReport(usageAndGroupsInfo, period, groups); + UsageAndGroupsInfo usageAndGroupsInfoForAllEntities = null; + if (countAllEntities) + { + usageAndGroupsInfoForAllEntities = gatherUsageAndGroups(groups, new Period(new Date(0), period.getUntil())); + } + String report = createReport(usageAndGroupsInfo, usageAndGroupsInfoForAllEntities, period, groups); sendReport(fromDateString, untilDateString, report); operationLog.info("Usage report created and sent."); } @@ -163,12 +183,18 @@ public class UsageReportingTask extends AbstractMaintenanceTask } } - private String createReport(UsageAndGroupsInfo usageAndGroupsInfo, Period period, List<String> groups) + private String createReport(UsageAndGroupsInfo usageAndGroupsInfo, + UsageAndGroupsInfo usageAndGroupsInfoForAllEntitiesOrNull, Period period, List<String> groups) { StringBuilder builder = new StringBuilder(); builder.append("period start" + DELIM + "period end" + DELIM + "group name" + DELIM + "number of users" + DELIM + "idle users" + DELIM + "number of new experiments" + DELIM + "number of new samples" + DELIM - + "number of new data sets\n"); + + "number of new data sets"); + if (usageAndGroupsInfoForAllEntitiesOrNull != null) + { + builder.append(DELIM).append("total number of entities"); + } + builder.append("\n"); Map<String, GroupInfo> groupInfos = initializeGroupInfos(usageAndGroupsInfo); Map<String, GroupInfo> individualInfos = new TreeMap<>(); for (String user : usageAndGroupsInfo.getUsageByUsersAndSpaces().keySet()) @@ -176,6 +202,47 @@ public class UsageReportingTask extends AbstractMaintenanceTask individualInfos.put(user, new GroupInfo(Arrays.asList(user))); } + handleUsageAndGroupInfos(usageAndGroupsInfo, groupInfos, individualInfos, new IUsageInfoHandler() + { + @Override + public void handleUsageInfo(GroupInfo groupInfo, String user, UsageInfo usageInfo, boolean groupAction) + { + userReportingType.handleUsageInfo(individualInfos.get(user), user, usageInfo, groupAction); + } + + @Override + public void handleGroupUsageInfo(GroupInfo groupInfo, String user, UsageInfo usageInfo) + { + groupInfo.handle(user, usageInfo); + } + }); + if (usageAndGroupsInfoForAllEntitiesOrNull != null) + { + handleUsageAndGroupInfos(usageAndGroupsInfoForAllEntitiesOrNull, groupInfos, individualInfos, new IUsageInfoHandler() + { + @Override + public void handleUsageInfo(GroupInfo groupInfo, String user, UsageInfo usageInfo, boolean groupAction) + { + groupInfo.countEntities(usageInfo); + } + + @Override + public void handleGroupUsageInfo(GroupInfo groupInfo, String user, UsageInfo usageInfo) + { + groupInfo.countEntities(usageInfo); + } + }); + } + + addInfos(builder, period, groupInfos, true); + + addInfos(builder, period, individualInfos, false); + return builder.toString(); + } + + private void handleUsageAndGroupInfos(UsageAndGroupsInfo usageAndGroupsInfo, Map<String, GroupInfo> groupInfos, + Map<String, GroupInfo> individualInfos, IUsageInfoHandler handler) + { Map<String, Set<String>> usersByGroups = usageAndGroupsInfo.getUsersByGroups(); for (Entry<String, Map<String, UsageInfo>> entry : usageAndGroupsInfo.getUsageByUsersAndSpaces().entrySet()) { @@ -184,7 +251,7 @@ public class UsageReportingTask extends AbstractMaintenanceTask { String space = entry2.getKey(); UsageInfo usageInfo = entry2.getValue(); - groupInfos.get("").handle(user, usageInfo); + handler.handleGroupUsageInfo(groupInfos.get(""), user, usageInfo); String[] spaceParts = space.split("_"); boolean groupAction = false; if (spaceParts.length > 1) @@ -194,18 +261,13 @@ public class UsageReportingTask extends AbstractMaintenanceTask Set<String> groupUsers = usersByGroups.get(group); if (groupInfo != null && groupUsers != null && groupUsers.contains(user)) { - groupInfo.handle(user, usageInfo); + handler.handleGroupUsageInfo(groupInfo, user, usageInfo); groupAction = true; } } - userReportingType.handleUsageInfo(individualInfos.get(user), user, usageInfo, groupAction); + handler.handleUsageInfo(individualInfos.get(user), user, usageInfo, groupAction); } } - - addInfos(builder, period, groupInfos, true); - - addInfos(builder, period, individualInfos, false); - return builder.toString(); } private Map<String, GroupInfo> initializeGroupInfos(UsageAndGroupsInfo usageAndGroupsInfo) @@ -237,7 +299,12 @@ public class UsageReportingTask extends AbstractMaintenanceTask builder.append(StringUtils.join(idleUsers, ' ')).append(DELIM); builder.append(info.getNumberOfNewExperiments()).append(DELIM); builder.append(info.getNumberOfNewSamples()).append(DELIM); - builder.append(info.getNumberOfNewDataSets()).append("\n"); + builder.append(info.getNumberOfNewDataSets()); + if (countAllEntities) + { + builder.append(DELIM).append(info.getNumberOfEntities()); + } + builder.append("\n"); } } } @@ -254,6 +321,8 @@ public class UsageReportingTask extends AbstractMaintenanceTask private int numberOfNewDataSets; + private int numberOfEntities; + public GroupInfo(Collection<String> users) { allUsers.addAll(users); @@ -270,6 +339,13 @@ public class UsageReportingTask extends AbstractMaintenanceTask numberOfNewDataSets += usageInfo.getNumberOfNewDataSets(); } + void countEntities(UsageInfo usageInfo) + { + numberOfEntities += usageInfo.getNumberOfNewDataSets(); + numberOfEntities += usageInfo.getNumberOfNewExperiments(); + numberOfEntities += usageInfo.getNumberOfNewSamples(); + } + int getNumberOfUsers() { return allUsers.size(); @@ -296,5 +372,10 @@ public class UsageReportingTask extends AbstractMaintenanceTask { return numberOfNewDataSets; } + + int getNumberOfEntities() + { + return numberOfEntities; + } } } diff --git a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/task/UsageReportingTaskTest.java b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/task/UsageReportingTaskTest.java index 6c7cba84f48..61d78ee0682 100644 --- a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/task/UsageReportingTaskTest.java +++ b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/task/UsageReportingTaskTest.java @@ -20,6 +20,7 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashSet; @@ -44,6 +45,7 @@ import ch.systemsx.cisd.common.logging.BufferedAppender; import ch.systemsx.cisd.common.mail.EMailAddress; import ch.systemsx.cisd.common.mail.IMailClient; import ch.systemsx.cisd.common.maintenance.MaintenanceTaskParameters; +import ch.systemsx.cisd.common.shared.basic.string.CommaSeparatedListBuilder; import ch.systemsx.cisd.common.test.RecordingMatcher; import ch.systemsx.cisd.openbis.generic.server.util.PluginUtils; import ch.systemsx.cisd.openbis.util.LogRecordingUtils; @@ -131,8 +133,8 @@ public class UsageReportingTaskTest extends AbstractFileSystemTestCase + "INFO OPERATION.UsageReportingTaskWithMocks - Gather usage information for the period from " + "1969-12-01 until 1970-01-01\n" + "INFO OPERATION.UsageReportingTaskWithMocks - Usage report created and sent.", logRecorder.getLogContent()); - assertPeriod("1969-12-01 00:00:00", "1970-01-01 00:00:00", task.period); - assertEquals("[B, A]", task.groups.toString()); + assertEquals("[1969-12-01 00:00:00, 1970-01-01 00:00:00]", task.periodListAsString()); + assertEquals("[[B, A]]", task.groupsListAsString()); assertRecorders(subjectRecorder, "Usage report for the period from 1969-12-01 until 1970-01-01", 2); assertRecorders(contentRecorder, "The usage report can be found in the attached TSV file.", 2); assertRecorders(fileNameRecorder, "usage_report_1969-12-01_1970-01-01.tsv", 2); @@ -183,8 +185,8 @@ public class UsageReportingTaskTest extends AbstractFileSystemTestCase + "INFO OPERATION.UsageReportingTaskWithMocks - Gather usage information for the period from " + "1969-12-21 until 1969-12-28\n" + "INFO OPERATION.UsageReportingTaskWithMocks - Usage report created and sent.", logRecorder.getLogContent()); - assertPeriod("1969-12-21 00:00:00", "1969-12-28 00:00:00", task.period); - assertEquals("[B, A]", task.groups.toString()); + assertEquals("[1969-12-21 00:00:00, 1969-12-28 00:00:00]", task.periodListAsString()); + assertEquals("[[B, A]]", task.groupsListAsString()); assertRecorders(subjectRecorder, "Usage report for the period from 1969-12-21 until 1969-12-28", 2); assertRecorders(contentRecorder, "The usage report can be found in the attached TSV file.", 2); assertRecorders(fileNameRecorder, "usage_report_1969-12-21_1969-12-28.tsv", 2); @@ -207,6 +209,67 @@ public class UsageReportingTaskTest extends AbstractFileSystemTestCase context.assertIsSatisfied(); } + @Test + public void testWeeklyGroupsAndAllUsersReportCountEntities() throws IOException + { + // Given + UsageReportingTaskWithMocks task = new UsageReportingTaskWithMocks(mailClient).time(new Date(1234567890)); + task.user("u1").space("A_X").newExperiments(2).newDataSets(1); + task.user("u1").space("C").newSamples(1); + task.user("u2").space("A_Y").newSamples(4); + task.user("u2").space("A_Z").newSamples(3); + task.user("u2").space("A1").newSamples(2).newDataSets(5); + task.user("u3").space("B1").newSamples(3); + task.user("u4"); + task.user("u5"); + task.group("A", "u1", "u4"); + task.group("B", "u2", "u3"); + task.next(); + task.user("u1").space("A_Y").newExperiments(5).newDataSets(6).newSamples(3); + task.user("u2").space("B_X").newExperiments(3).newDataSets(7).newSamples(13); + task.user("u3").space("B_Y").newSamples(3).newDataSets(11); + task.user("u3").space("C").newExperiments(2).newDataSets(17); + task.user("u4").space("A_Z").newExperiments(5).newDataSets(16); + FileUtilities.writeToFile(configFile, ""); + properties.setProperty(MaintenanceTaskParameters.INTERVAL_KEY, "7 d"); + properties.setProperty(UsageReportingTask.USER_REPORTING_KEY, UsageReportingTask.UserReportingType.ALL.name()); + properties.setProperty(UsageReportingTask.COUNT_ALL_ENTITIES_KEY, Boolean.toString(true)); + task.setUp("", properties); + FileUtilities.writeToFile(configFile, "{\"groups\": [{\"key\":\"B\"}, {\"key\":\"A\"}]}"); + + // When + task.execute(); + + // Then + assertEquals("INFO OPERATION.UsageReportingTaskWithMocks - Setup plugin \n" + + "INFO OPERATION.UsageReportingTaskWithMocks - Plugin '' initialized. Configuration file: " + configFile.getAbsolutePath() + "\n" + + "INFO OPERATION.UsageReportingTaskWithMocks - Gather usage information for the period from " + + "1970-01-04 until 1970-01-11\n" + + "INFO OPERATION.UsageReportingTaskWithMocks - Usage report created and sent.", logRecorder.getLogContent()); + assertEquals("[1970-01-04 00:00:00, 1970-01-11 00:00:00], [1970-01-01 01:00:00, 1970-01-11 00:00:00]", task.periodListAsString()); + assertEquals("[[B, A], [B, A]]", task.groupsListAsString()); + assertRecorders(subjectRecorder, "Usage report for the period from 1970-01-04 until 1970-01-11", 2); + assertRecorders(contentRecorder, "The usage report can be found in the attached TSV file.", 2); + assertRecorders(fileNameRecorder, "usage_report_1970-01-04_1970-01-11.tsv", 2); + assertEquals("EmailAddress{email=a1@bc.de}", recipientRecorder.getRecordedObjects().get(0)[0].toString()); + assertEquals("EmailAddress{email=a2@bc.de}", recipientRecorder.getRecordedObjects().get(1)[0].toString()); + assertEquals(2, recipientRecorder.getRecordedObjects().size()); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + attachmentRecorder.getRecordedObjects().get(0).writeTo(baos); + assertEquals("period start" + D + "period end" + D + "group name" + D + "number of users" + D + "idle users" + D + + "number of new experiments" + D + "number of new samples" + D + "number of new data sets" + D + "total number of entities\n" + + "1970-01-04 00:00:00" + D + "1970-01-11 00:00:00" + D + "" + D + "5" + D + "u4 u5" + D + "2" + D + "13" + D + "6" + D + "91\n" + + "1970-01-04 00:00:00" + D + "1970-01-11 00:00:00" + D + "A" + D + "2" + D + "u4" + D + "2" + D + "0" + D + "1" + D + "35\n" + + "1970-01-04 00:00:00" + D + "1970-01-11 00:00:00" + D + "B" + D + "2" + D + "u2 u3" + D + "0" + D + "0" + D + "0" + D + "37\n" + + "1970-01-04 00:00:00" + D + "1970-01-11 00:00:00" + D + "u1" + D + "1" + D + D + "2" + D + "1" + D + "1" + D + "14\n" + + "1970-01-04 00:00:00" + D + "1970-01-11 00:00:00" + D + "u2" + D + "1" + D + D + "0" + D + "9" + D + "5" + D + "23\n" + + "1970-01-04 00:00:00" + D + "1970-01-11 00:00:00" + D + "u3" + D + "1" + D + D + "0" + D + "3" + D + "0" + D + "33\n", baos.toString()); + assertEquals("text/plain", attachmentRecorder.getRecordedObjects().get(0).getContentType()); + assertEquals("text/plain", attachmentRecorder.getRecordedObjects().get(1).getContentType()); + assertEquals(2, attachmentRecorder.getRecordedObjects().size()); + context.assertIsSatisfied(); + } + @Test public void testDailyGroupsAndOutsideUsersReport() throws IOException { @@ -238,8 +301,8 @@ public class UsageReportingTaskTest extends AbstractFileSystemTestCase + "INFO OPERATION.UsageReportingTaskWithMocks - Gather usage information for the period from " + "1969-12-31 until 1970-01-01\n" + "INFO OPERATION.UsageReportingTaskWithMocks - Usage report created and sent.", logRecorder.getLogContent()); - assertPeriod("1969-12-31 00:00:00", "1970-01-01 00:00:00", task.period); - assertEquals("[B, A]", task.groups.toString()); + assertEquals("[1969-12-31 00:00:00, 1970-01-01 00:00:00]", task.periodListAsString()); + assertEquals("[[B, A]]", task.groupsListAsString()); assertRecorders(subjectRecorder, "Usage report for the period from 1969-12-31 until 1970-01-01", 2); assertRecorders(contentRecorder, "The usage report can be found in the attached TSV file.", 2); assertRecorders(fileNameRecorder, "usage_report_1969-12-31_1970-01-01.tsv", 2); @@ -285,8 +348,8 @@ public class UsageReportingTaskTest extends AbstractFileSystemTestCase + "INFO OPERATION.UsageReportingTaskWithMocks - Gather usage information for the period from " + "1969-12-31 until 1970-01-01\n" + "INFO OPERATION.UsageReportingTaskWithMocks - Usage report created and sent.", logRecorder.getLogContent()); - assertPeriod("1969-12-31 00:00:00", "1970-01-01 00:00:00", task.period); - assertEquals(null, task.groups); + assertEquals("[1969-12-31 00:00:00, 1970-01-01 00:00:00]", task.periodListAsString()); + assertEquals("[null]", task.groupsListAsString()); assertRecorders(subjectRecorder, "Usage report for the period from 1969-12-31 until 1970-01-01", 2); assertRecorders(contentRecorder, "The usage report can be found in the attached TSV file.", 2); assertRecorders(fileNameRecorder, "usage_report_1969-12-31_1970-01-01.tsv", 2); @@ -332,8 +395,8 @@ public class UsageReportingTaskTest extends AbstractFileSystemTestCase + "INFO OPERATION.UsageReportingTaskWithMocks - Gather usage information for the period from " + "1969-12-31 until 1970-01-01\n" + "INFO OPERATION.UsageReportingTaskWithMocks - Usage report created and sent.", logRecorder.getLogContent()); - assertPeriod("1969-12-31 00:00:00", "1970-01-01 00:00:00", task.period); - assertEquals(null, task.groups); + assertEquals("[1969-12-31 00:00:00, 1970-01-01 00:00:00]", task.periodListAsString()); + assertEquals("[null]", task.groupsListAsString()); assertRecorders(subjectRecorder, "Usage report for the period from 1969-12-31 until 1970-01-01", 2); assertRecorders(contentRecorder, "The usage report can be found in the attached TSV file.", 2); assertRecorders(fileNameRecorder, "usage_report_1969-12-31_1970-01-01.tsv", 2); @@ -363,31 +426,32 @@ public class UsageReportingTaskTest extends AbstractFileSystemTestCase } } - private void assertPeriod(String expectedFrom, String expectedUntil, Period period) - { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - assertEquals(expectedFrom, dateFormat.format(period.getFrom())); - assertEquals(expectedUntil, dateFormat.format(period.getUntil())); - } - private static final class UsageReportingTaskWithMocks extends UsageReportingTask { private final IMailClient mailClient; private Date actualTimeStamp; - private Map<String, Map<String, UsageInfo>> usageByUsersAndSpaces = new TreeMap<>(); + private List<Map<String, Map<String, UsageInfo>>> listOfUsageByUsersAndSpaces = new ArrayList<>(); private Map<String, Set<String>> usersByGroups = new TreeMap<>(); - private List<String> groups; + private List<List<String>> groupsList = new ArrayList<>(); - private Period period; + private List<Period> periodList = new ArrayList<>(); + + private int index; UsageReportingTaskWithMocks(IMailClient mailClient) { this.mailClient = mailClient; - + next(); + } + + void next() + { + listOfUsageByUsersAndSpaces.add(0, new TreeMap<>()); + index = listOfUsageByUsersAndSpaces.size(); } UsageReportingTaskWithMocks time(Date actualTimeStamp) @@ -398,11 +462,11 @@ public class UsageReportingTaskTest extends AbstractFileSystemTestCase UsageBySpacesBuilder user(String user) { - Map<String, UsageInfo> usageBySpaces = usageByUsersAndSpaces.get(user); + Map<String, UsageInfo> usageBySpaces = listOfUsageByUsersAndSpaces.get(0).get(user); if (usageBySpaces == null) { usageBySpaces = new TreeMap<>(); - usageByUsersAndSpaces.put(user, usageBySpaces); + listOfUsageByUsersAndSpaces.get(0).put(user, usageBySpaces); } return new UsageBySpacesBuilder(usageBySpaces); } @@ -422,9 +486,9 @@ public class UsageReportingTaskTest extends AbstractFileSystemTestCase @Override protected UsageAndGroupsInfo gatherUsageAndGroups(List<String> groups, Period period) { - this.groups = groups; - this.period = period; - return new UsageAndGroupsInfo(usageByUsersAndSpaces, usersByGroups); + groupsList.add(groups); + periodList.add(period); + return new UsageAndGroupsInfo(listOfUsageByUsersAndSpaces.get(--index), usersByGroups); } @Override @@ -432,6 +496,22 @@ public class UsageReportingTaskTest extends AbstractFileSystemTestCase { return mailClient; } + + String groupsListAsString() + { + return groupsList.toString(); + } + + String periodListAsString() + { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + CommaSeparatedListBuilder builder = new CommaSeparatedListBuilder(); + for (Period period : periodList) + { + builder.append("[" + dateFormat.format(period.getFrom()) + ", " + dateFormat.format(period.getUntil()) + "]"); + } + return builder.toString(); + } } private static final class UsageBySpacesBuilder -- GitLab