SDAP-61 Added a test job for climatology ingestion.
authorfgreg <fgreguska+github@gmail.com>
Tue, 17 Apr 2018 22:55:45 +0000 (15:55 -0700)
committerGitHub <noreply@github.com>
Tue, 17 Apr 2018 22:55:45 +0000 (15:55 -0700)
src/test/java/org/apache/sdap/ningester/processors/AddDayOfYearAttributeTest.java
src/test/java/org/apache/sdap/ningester/processors/AddTimeFromGranuleNameTest.java
src/test/resources/granules/001.L4_5day_avhrr_clim_sst_pixelMean.nc [new file with mode: 0644]
src/testJobs/java/org/apache/sdap/ningester/testjobs/AvhrrClimJobTest.java [new file with mode: 0644]
src/testJobs/resources/testjobs/AvhrrClimJobTest.yml [new file with mode: 0644]

index d43e0ec..a6aef61 100644 (file)
@@ -36,8 +36,8 @@ public class AddDayOfYearAttributeTest {
 
     @Test
     public void testSuccessfulMatch() {
-        String regex = "^(\\d{3})";
-        String granuleName = "001.L4_5day_avhrr_clim_sst_pixelMean.nc";
+        String regex = "^.*(\\d{3})";
+        String granuleName = "file:/some/path/001.L4_5day_avhrr_clim_sst_pixelMean.nc";
         NexusTile nexusTile = NexusTile.newBuilder().setSummary(
                 TileSummary.newBuilder()
                         .setGranule(granuleName)
index 4842290..86ba34d 100644 (file)
@@ -43,7 +43,7 @@ public class AddTimeFromGranuleNameTest {
         String regex = "^.*(\\d{7})\\.";
         String dateFormat = "yyyyDDD";
 
-        String granuleName = "A2012001.L3m_DAY_NSST_sst_4km.nc";
+        String granuleName = "file:/some/path/A2012001.L3m_DAY_NSST_sst_4km.nc";
         Long expectedTime = 1325376000L; // 01/01/2012 00:00:00 in epoch time
         NexusTile nexusTile = NexusTile.newBuilder().setSummary(
                 TileSummary.newBuilder()
@@ -68,6 +68,35 @@ public class AddTimeFromGranuleNameTest {
     }
 
     @Test
+    public void testSuccessfulMatchAvhrr() {
+        String regex = "^.*(\\d{3})\\.";
+        String dateFormat = "DDD";
+
+        String granuleName = "file:/some/path/005.L4_5day_avhrr_clim_sst_pixelMean.nc";
+        Long expectedTime = 345600L; // 01/05/1979 00:00:00 in epoch time
+        NexusTile nexusTile = NexusTile.newBuilder().setSummary(
+                TileSummary.newBuilder()
+                        .setGranule(granuleName)
+                        .build()
+        ).setTile(
+                TileData.newBuilder()
+                        .setGridTile(
+                                GridTile.newBuilder(
+
+                                ).build()
+                        ).build()
+        ).build();
+
+        AddTimeFromGranuleName processor = new AddTimeFromGranuleName(regex, dateFormat);
+
+        NexusTile result = processor.setTimeFromGranuleName(nexusTile);
+
+        assertThat(result.getTile().getGridTile().getTime(), is(expectedTime));
+        assertThat(result.getSummary().getStats().getMinTime(), is(expectedTime));
+        assertThat(result.getSummary().getStats().getMaxTime(), is(expectedTime));
+    }
+
+    @Test
     public void testUnparseable() {
         String regex = "^.*(\\d{7})\\.";
         String dateFormat = "yyyyDDDss";
diff --git a/src/test/resources/granules/001.L4_5day_avhrr_clim_sst_pixelMean.nc b/src/test/resources/granules/001.L4_5day_avhrr_clim_sst_pixelMean.nc
new file mode 100644 (file)
index 0000000..dc4e287
Binary files /dev/null and b/src/test/resources/granules/001.L4_5day_avhrr_clim_sst_pixelMean.nc differ
diff --git a/src/testJobs/java/org/apache/sdap/ningester/testjobs/AvhrrClimJobTest.java b/src/testJobs/java/org/apache/sdap/ningester/testjobs/AvhrrClimJobTest.java
new file mode 100644 (file)
index 0000000..c0e52a5
--- /dev/null
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.sdap.ningester.testjobs;
+
+import org.apache.sdap.ningester.configuration.properties.ApplicationProperties;
+import org.apache.sdap.ningester.configuration.properties.DatasourceProperties;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.batch.core.JobExecution;
+import org.springframework.batch.core.JobParameters;
+import org.springframework.batch.core.JobParametersBuilder;
+import org.springframework.batch.core.StepExecution;
+import org.springframework.batch.test.JobLauncherTestUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.context.TestConfiguration;
+import org.springframework.context.annotation.Bean;
+import org.springframework.data.cassandra.core.CassandraTemplate;
+import org.springframework.data.solr.core.SolrTemplate;
+import org.springframework.data.solr.core.query.SimpleQuery;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.TestPropertySource;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import static org.apache.sdap.ningester.testjobs.TestUtils.assertEqualsEventually;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest
+@TestPropertySource(properties = {"spring.config.location = classpath:testjobs/AvhrrClimJobTest.yml"})
+@ActiveProfiles({"test", "cassandra", "solr"})
+public class AvhrrClimJobTest {
+
+    @Autowired
+    JobLauncherTestUtils jobLauncherTestUtils;
+    @Autowired
+    CassandraTemplate cassandraTemplate;
+    @Autowired
+    SolrTemplate solrTemplate;
+    @Autowired
+    DatasourceProperties datasourceProperties;
+    @Autowired
+    ApplicationProperties applicationProperties;
+
+    @Before
+    public void emptyDatabase() {
+        solrTemplate.delete(datasourceProperties.getSolrStore().getCollection(), new SimpleQuery("*:*"));
+        cassandraTemplate.truncate(datasourceProperties.getCassandraStore().getTableName());
+    }
+
+    @Test
+    public void testJobCompletes() throws Exception {
+
+        JobParameters jobParameters = new JobParametersBuilder()
+                .addString("granule", "classpath:granules/001.L4_5day_avhrr_clim_sst_pixelMean.nc")
+                .toJobParameters();
+
+        JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters);
+
+        assertThat(jobExecution.getExitStatus().getExitCode(), is("COMPLETED"));
+        StepExecution stepExecution = jobExecution.getStepExecutions().iterator().next();
+        assertThat(stepExecution.getReadCount(), is(1296));
+        assertThat(stepExecution.getWriteCount(), is(1053));
+        assertThat(stepExecution.getFilterCount(), is(243));
+
+        assertEqualsEventually(1053L,
+                () -> solrTemplate.count(datasourceProperties.getSolrStore().getCollection(),
+                        new SimpleQuery("dataset_s: " + applicationProperties.getAddDatasetName().getDatasetName())),
+                3);
+
+        long cassandraCount = cassandraTemplate.count(datasourceProperties.getCassandraStore().getTableName());
+
+        assertThat(cassandraCount, is(1053L));
+    }
+
+    @TestConfiguration
+    static class NingesterApplicationTestsConfig {
+
+        @Bean
+        JobLauncherTestUtils jobLauncherTestUtils() {
+            return new JobLauncherTestUtils();
+        }
+
+    }
+
+
+}
diff --git a/src/testJobs/resources/testjobs/AvhrrClimJobTest.yml b/src/testJobs/resources/testjobs/AvhrrClimJobTest.yml
new file mode 100644 (file)
index 0000000..36e52f8
--- /dev/null
@@ -0,0 +1,70 @@
+# Tile Slicer Config
+ningester:
+  tile_slicer: sliceFileByTilesDesired
+  sliceFileByTilesDesired:
+    tilesDesired: 1296
+    timeDimension: time
+    dimensions:
+      - lat
+      - lon
+---
+# Tile processors configuration
+ningester:
+  tile_processors:
+    - pythonChainProcessor
+    - generateTileId
+    - addTimeFromGranuleName
+    - addDayOfYearAttribute
+    - addDatasetName
+  pythonChainProcessor:
+    enabled:
+    processor_list:
+      -
+        name: GridReadingProcessor
+        config:
+          latitude: lat
+          longitude: lon
+          time: time
+          variable_to_read: analysed_sst
+      -
+        name: EmptyTileFilter
+      -
+        name: KelvinToCelsius
+      -
+        name: TileSummarizingProcessor
+        config:
+          stored_var_name: analysed_sst
+  generateTileId:
+    enabled:
+  addTimeFromGranuleName:
+    enabled:
+    regex: ^.*(\d{3})\.
+    dateFormat: DDD
+  addDayOfYearAttribute:
+    enabled:
+    regex: ^.*(\d{3})\.
+  addDatasetName:
+    enabled:
+    datasetName: AVHRR_OI_L4_GHRSST_NCEI_CLIM
+---
+# Tile writer configuration
+ningester:
+  tile_writer:
+    data_store: cassandraStore
+    metadata_store: solrStore
+---
+# Connection settings for the test profile
+spring:
+    profiles:
+      - test
+      - local
+    data:
+      cassandra:
+        keyspaceName: nexustiles
+        contactPoints: 127.0.0.1
+      solr:
+        host: http://127.0.0.1:8983/solr/
+
+datasource:
+  solrStore:
+    collection: nexustiles
\ No newline at end of file