1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.timezone.location.data_pipeline.steps;
18 
19 import static com.google.common.collect.ImmutableSet.toImmutableSet;
20 
21 import com.android.timezone.location.common.LicenseSupport;
22 import com.android.timezone.location.common.LicenseSupport.License;
23 import com.android.timezone.location.data_pipeline.steps.Types.ProtoStorageFormat;
24 import com.android.timezone.location.data_pipeline.steps.Types.TzS2Polygons;
25 import com.android.timezone.location.data_pipeline.util.NamedFuture;
26 
27 import com.beust.jcommander.JCommander;
28 import com.beust.jcommander.Parameter;
29 import com.beust.jcommander.converters.FileConverter;
30 import com.fasterxml.jackson.databind.ObjectMapper;
31 import com.google.common.collect.ImmutableSet;
32 import com.google.common.geometry.S2LatLng;
33 import com.google.common.geometry.S2Loop;
34 import com.google.common.geometry.S2Point;
35 import com.google.common.geometry.S2Polygon;
36 import org.geojson.Feature;
37 import org.geojson.FeatureCollection;
38 import org.geojson.Geometry;
39 import org.geojson.LngLatAlt;
40 import org.geojson.MultiPolygon;
41 import org.geojson.Polygon;
42 
43 import java.io.BufferedInputStream;
44 import java.io.File;
45 import java.io.FileInputStream;
46 import java.io.IOException;
47 import java.io.InputStream;
48 import java.util.ArrayList;
49 import java.util.Arrays;
50 import java.util.Collections;
51 import java.util.HashSet;
52 import java.util.List;
53 import java.util.Objects;
54 import java.util.Set;
55 import java.util.concurrent.ExecutionException;
56 import java.util.concurrent.ExecutorService;
57 import java.util.concurrent.Executors;
58 import java.util.concurrent.Future;
59 import java.util.concurrent.TimeUnit;
60 
61 /**
62  * An executable class that takes a geojson file produced by the external/timezone-boundary-builder
63  * project and converts all the time zone polygons into {@link Types.TzS2Polygons}, one per time
64  * zone ID encountered. The resulting files can be discovered using
65  * {@link TzS2Polygons#listFiles(File, ProtoStorageFormat)} and loaded individually using
66  * {@link TzS2Polygons#load(File, ProtoStorageFormat)}. See {@link #main(String[])} for usage.
67  */
68 public final class GeoJsonTzToTzS2Polygons {
69 
70     private final File mInputFile;
71 
72     private final ExecutorService mExecutorService;
73 
74     private final File mOutputDir;
75 
76     private final ProtoStorageFormat mProtoStorageFormat;
77 
78     private final Set<String> mTzIds;
79 
GeoJsonTzToTzS2Polygons( File inputFile, ExecutorService executorService, File outputDir, ProtoStorageFormat protoStorageFormat, Set<String> tzIds)80     private GeoJsonTzToTzS2Polygons(
81             File inputFile, ExecutorService executorService, File outputDir,
82             ProtoStorageFormat protoStorageFormat,
83             Set<String> tzIds) {
84         this.mInputFile = Objects.requireNonNull(inputFile);
85         this.mExecutorService = Objects.requireNonNull(executorService);
86         this.mOutputDir = Objects.requireNonNull(outputDir);
87         this.mProtoStorageFormat = Objects.requireNonNull(protoStorageFormat);
88         this.mTzIds = Objects.requireNonNull(tzIds);
89     }
90 
91     private static class Arguments {
92 
93         @Parameter(names = "--geo-json",
94                 description = "The input geojson file to parse",
95                 required = true,
96                 converter = FileConverter.class)
97         File geoJsonFile;
98 
99         @Parameter(names = "--num-threads",
100                 description = "The number of threads to use",
101                 required = true)
102         int numThreads;
103 
104         @Parameter(names = "--output",
105                 description = "The output directory",
106                 required = true,
107                 converter = FileConverter.class)
108         File outputDir;
109 
110         @Parameter(names = "--tz-ids",
111                 description = "Comma separated list of time zones to build S2Polygons for")
112         String tzIds;
113 
tzIds()114         Set<String> tzIds() {
115             return tzIds == null
116                     ? ImmutableSet.of()
117                     : Arrays.stream(tzIds.split(","))
118                             .filter(tzId -> !tzId.isEmpty())
119                             .collect(toImmutableSet());
120         }
121     }
122 
123     /**
124      * See {@link GeoJsonTzToTzS2Polygons} for the purpose of this class.
125      */
main(String[] args)126     public static void main(String[] args) throws Exception {
127         Arguments arguments = new Arguments();
128         JCommander.newBuilder()
129                 .addObject(arguments)
130                 .build()
131                 .parse(args);
132 
133         File inputFile = arguments.geoJsonFile;
134         int threads = arguments.numThreads;
135         File outputDir = arguments.outputDir;
136         Set<String> tzIds = arguments.tzIds();
137         ProtoStorageFormat protoStorageFormat = Types.DEFAULT_PROTO_STORAGE_FORMAT;
138 
139         outputDir.mkdirs();
140 
141         ExecutorService executorService = Executors.newFixedThreadPool(threads);
142 
143         GeoJsonTzToTzS2Polygons converter = new GeoJsonTzToTzS2Polygons(
144                 inputFile, executorService, outputDir, protoStorageFormat, tzIds);
145 
146         try {
147             converter.execute();
148         } finally {
149             System.out.println("Waiting for shutdown");
150             executorService.shutdown();
151             executorService.awaitTermination(5, TimeUnit.SECONDS);
152         }
153     }
154 
execute()155     private void execute() throws Exception {
156         FeatureCollection featureCollection = loadFeatures(mInputFile);
157 
158         LicenseSupport.copyLicenseFile(mInputFile.getParentFile(), mOutputDir);
159 
160         if (!mTzIds.isEmpty()) {
161             System.out.println("Building polygons for " + mTzIds);
162         }
163 
164         List<Feature> features = featureCollection.getFeatures();
165         List<NamedFuture<TzS2Polygons>> futures = new ArrayList<>(features.size());
166         for (Feature feature : features) {
167             String tzId = (String) feature.getProperties().get("tzid");
168             if (!mTzIds.isEmpty() && !mTzIds.contains(tzId)) {
169                 continue;
170             }
171             System.out.println("Submitting " + tzId + " ...");
172             Future<TzS2Polygons> future = mExecutorService.submit(() -> processFeature(feature));
173             NamedFuture<TzS2Polygons> namedFuture = new NamedFuture<>(tzId, future);
174             futures.add(namedFuture);
175         }
176 
177         Set<String> knownTzIds = new HashSet<>(features.size());
178         for (NamedFuture<TzS2Polygons> future : futures) {
179             try {
180                 if (!future.isDone()) {
181                     System.out.println("Waiting for " + future.getName());
182                 }
183                 TzS2Polygons tzPolygons = future.get();
184                 String tzId = tzPolygons.tzId;
185                 if (knownTzIds.contains(tzId)) {
186                     throw new IllegalStateException("Multiple entries found for: " + tzId);
187                 }
188                 knownTzIds.add(tzId);
189 
190                 String fileSuffix = TzS2Polygons.getFileSuffix(mProtoStorageFormat);
191                 File outputFile = TzIds.createFile(mOutputDir, tzPolygons.tzId, fileSuffix);
192                 TzS2Polygons.store(tzPolygons, outputFile, mProtoStorageFormat, License.ODBL);
193             } catch (InterruptedException | ExecutionException e) {
194                 throw new RuntimeException(e);
195             }
196         }
197     }
198 
processFeature(Feature feature)199     private static TzS2Polygons processFeature(Feature feature) {
200         String tzId = (String) feature.getProperties().get("tzid");
201         System.out.println("Converting " + tzId + " to S2 geometry...");
202         Geometry geometry = (Geometry) feature.getGeometry();
203         List<Polygon> polygons;
204         if (geometry instanceof Polygon) {
205             polygons = Collections.singletonList((Polygon) geometry);
206         } else if (geometry instanceof MultiPolygon) {
207             MultiPolygon multiPolygon = (MultiPolygon) geometry;
208             polygons = new ArrayList<>();
209             for (List<List<LngLatAlt>> polygonCoordinates : multiPolygon.getCoordinates()) {
210                 Polygon polygon = new Polygon();
211                 for (List<LngLatAlt> loop : polygonCoordinates) {
212                     polygon.add(loop);
213                 }
214                 polygons.add(polygon);
215             }
216         } else {
217             throw new IllegalStateException(geometry.getClass().toString());
218         }
219 
220         List<S2Polygon> s2PolygonList = new ArrayList<>();
221         for (Polygon polygon : polygons) {
222             S2Polygon s2Polygon = createS2Polygon(polygon);
223             s2PolygonList.add(s2Polygon);
224         }
225         return new TzS2Polygons(tzId, s2PolygonList);
226     }
227 
loadFeatures(File file)228     private static FeatureCollection loadFeatures(File file) throws IOException {
229         FeatureCollection featureCollection;
230         final int bufferSize = 256 * 1024;
231         try (InputStream inputStream = new BufferedInputStream(
232                 new FileInputStream(file), bufferSize)) {
233             featureCollection = new ObjectMapper().readValue(inputStream, FeatureCollection.class);
234             System.out.println("Features read: " + featureCollection.getFeatures().size());
235         }
236         return featureCollection;
237     }
238 
createS2Polygon(Polygon polygon)239     private static S2Polygon createS2Polygon(Polygon polygon) {
240         List<S2Loop> s2Loops = new ArrayList<>();
241         for (List<LngLatAlt> loop : polygon.getCoordinates()) {
242             S2Loop s2Loop = createS2Loop(loop);
243             // There's a problem with geojson in that the order of coordinates is undefined,
244             // while S2 requires them in CCW order. normalize() should address this.
245             s2Loop.normalize();
246             if (!s2Loop.isValid()) {
247                 throw new IllegalStateException("Invalid loop from polygon:" + polygon);
248             }
249             s2Loops.add(s2Loop);
250         }
251         return new S2Polygon(s2Loops);
252     }
253 
createS2Loop(List<LngLatAlt> loop)254     private static S2Loop createS2Loop(List<LngLatAlt> loop) {
255         List<S2Point> s2Points = new ArrayList<>();
256         for (LngLatAlt point : loop) {
257             S2LatLng s2LatLng = S2LatLng.fromDegrees(point.getLatitude(), point.getLongitude());
258             S2Point s2Point = s2LatLng.toPoint();
259             s2Points.add(s2Point);
260         }
261         // It appears to be normal for geojson to have the first and last point be the same. S2Loop
262         // considers this invalid so we remove it.
263         if (s2Points.get(0).equals(s2Points.get(s2Points.size() - 1))) {
264             s2Points.remove(s2Points.size() - 1);
265         } else {
266             // If you see this look for other changes in the input format.
267             throw new IllegalStateException(
268                     "GeoJSON loop did not start and end with the same point");
269         }
270         return new S2Loop(s2Points);
271     }
272 }
273