001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.InputStream; 007import java.io.InputStreamReader; 008import java.text.MessageFormat; 009import java.util.ArrayList; 010import java.util.Collection; 011import java.util.List; 012import java.util.regex.Matcher; 013import java.util.regex.Pattern; 014 015import javax.xml.stream.Location; 016import javax.xml.stream.XMLInputFactory; 017import javax.xml.stream.XMLStreamConstants; 018import javax.xml.stream.XMLStreamException; 019import javax.xml.stream.XMLStreamReader; 020 021import org.openstreetmap.josm.Main; 022import org.openstreetmap.josm.data.Bounds; 023import org.openstreetmap.josm.data.DataSource; 024import org.openstreetmap.josm.data.coor.LatLon; 025import org.openstreetmap.josm.data.osm.Changeset; 026import org.openstreetmap.josm.data.osm.DataSet; 027import org.openstreetmap.josm.data.osm.Node; 028import org.openstreetmap.josm.data.osm.NodeData; 029import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 030import org.openstreetmap.josm.data.osm.PrimitiveData; 031import org.openstreetmap.josm.data.osm.Relation; 032import org.openstreetmap.josm.data.osm.RelationData; 033import org.openstreetmap.josm.data.osm.RelationMemberData; 034import org.openstreetmap.josm.data.osm.Tagged; 035import org.openstreetmap.josm.data.osm.User; 036import org.openstreetmap.josm.data.osm.Way; 037import org.openstreetmap.josm.data.osm.WayData; 038import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 039import org.openstreetmap.josm.gui.progress.ProgressMonitor; 040import org.openstreetmap.josm.tools.CheckParameterUtil; 041import org.openstreetmap.josm.tools.date.DateUtils; 042 043/** 044 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it. 045 * 046 * For each xml element, there is a dedicated method. 047 * The XMLStreamReader cursor points to the start of the element, when the method is 048 * entered, and it must point to the end of the same element, when it is exited. 049 */ 050public class OsmReader extends AbstractReader { 051 052 protected XMLStreamReader parser; 053 054 protected boolean cancel; 055 056 /** Used by plugins to register themselves as data postprocessors. */ 057 private static volatile List<OsmServerReadPostprocessor> postprocessors; 058 059 /** Register a new postprocessor. 060 * @param pp postprocessor 061 * @see #deregisterPostprocessor 062 */ 063 public static void registerPostprocessor(OsmServerReadPostprocessor pp) { 064 if (postprocessors == null) { 065 postprocessors = new ArrayList<>(); 066 } 067 postprocessors.add(pp); 068 } 069 070 /** 071 * Deregister a postprocessor previously registered with {@link #registerPostprocessor}. 072 * @param pp postprocessor 073 * @see #registerPostprocessor 074 */ 075 public static void deregisterPostprocessor(OsmServerReadPostprocessor pp) { 076 if (postprocessors != null) { 077 postprocessors.remove(pp); 078 } 079 } 080 081 /** 082 * constructor (for private and subclasses use only) 083 * 084 * @see #parseDataSet(InputStream, ProgressMonitor) 085 */ 086 protected OsmReader() { 087 // Restricts visibility 088 } 089 090 protected void setParser(XMLStreamReader parser) { 091 this.parser = parser; 092 } 093 094 protected void throwException(String msg, Throwable th) throws XMLStreamException { 095 throw new OsmParsingException(msg, parser.getLocation(), th); 096 } 097 098 protected void throwException(String msg) throws XMLStreamException { 099 throw new OsmParsingException(msg, parser.getLocation()); 100 } 101 102 protected void parse() throws XMLStreamException { 103 int event = parser.getEventType(); 104 while (true) { 105 if (event == XMLStreamConstants.START_ELEMENT) { 106 parseRoot(); 107 } else if (event == XMLStreamConstants.END_ELEMENT) 108 return; 109 if (parser.hasNext()) { 110 event = parser.next(); 111 } else { 112 break; 113 } 114 } 115 parser.close(); 116 } 117 118 protected void parseRoot() throws XMLStreamException { 119 if ("osm".equals(parser.getLocalName())) { 120 parseOsm(); 121 } else { 122 parseUnknown(); 123 } 124 } 125 126 private void parseOsm() throws XMLStreamException { 127 String v = parser.getAttributeValue(null, "version"); 128 if (v == null) { 129 throwException(tr("Missing mandatory attribute ''{0}''.", "version")); 130 } 131 if (!"0.6".equals(v)) { 132 throwException(tr("Unsupported version: {0}", v)); 133 } 134 ds.setVersion(v); 135 String upload = parser.getAttributeValue(null, "upload"); 136 if (upload != null) { 137 ds.setUploadDiscouraged(!Boolean.parseBoolean(upload)); 138 } 139 String generator = parser.getAttributeValue(null, "generator"); 140 Long uploadChangesetId = null; 141 if (parser.getAttributeValue(null, "upload-changeset") != null) { 142 uploadChangesetId = getLong("upload-changeset"); 143 } 144 while (true) { 145 int event = parser.next(); 146 147 if (cancel) { 148 cancel = false; 149 throw new OsmParsingCanceledException(tr("Reading was canceled"), parser.getLocation()); 150 } 151 152 if (event == XMLStreamConstants.START_ELEMENT) { 153 switch (parser.getLocalName()) { 154 case "bounds": 155 parseBounds(generator); 156 break; 157 case "node": 158 parseNode(); 159 break; 160 case "way": 161 parseWay(); 162 break; 163 case "relation": 164 parseRelation(); 165 break; 166 case "changeset": 167 parseChangeset(uploadChangesetId); 168 break; 169 default: 170 parseUnknown(); 171 } 172 } else if (event == XMLStreamConstants.END_ELEMENT) 173 return; 174 } 175 } 176 177 private void parseBounds(String generator) throws XMLStreamException { 178 String minlon = parser.getAttributeValue(null, "minlon"); 179 String minlat = parser.getAttributeValue(null, "minlat"); 180 String maxlon = parser.getAttributeValue(null, "maxlon"); 181 String maxlat = parser.getAttributeValue(null, "maxlat"); 182 String origin = parser.getAttributeValue(null, "origin"); 183 if (minlon != null && maxlon != null && minlat != null && maxlat != null) { 184 if (origin == null) { 185 origin = generator; 186 } 187 Bounds bounds = new Bounds( 188 Double.parseDouble(minlat), Double.parseDouble(minlon), 189 Double.parseDouble(maxlat), Double.parseDouble(maxlon)); 190 if (bounds.isOutOfTheWorld()) { 191 Bounds copy = new Bounds(bounds); 192 bounds.normalize(); 193 Main.info("Bbox " + copy + " is out of the world, normalized to " + bounds); 194 } 195 DataSource src = new DataSource(bounds, origin); 196 ds.dataSources.add(src); 197 } else { 198 throwException(tr("Missing mandatory attributes on element ''bounds''. " + 199 "Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.", 200 minlon, minlat, maxlon, maxlat, origin 201 )); 202 } 203 jumpToEnd(); 204 } 205 206 protected Node parseNode() throws XMLStreamException { 207 NodeData nd = new NodeData(); 208 String lat = parser.getAttributeValue(null, "lat"); 209 String lon = parser.getAttributeValue(null, "lon"); 210 if (lat != null && lon != null) { 211 nd.setCoor(new LatLon(Double.parseDouble(lat), Double.parseDouble(lon))); 212 } 213 readCommon(nd); 214 Node n = new Node(nd.getId(), nd.getVersion()); 215 n.setVisible(nd.isVisible()); 216 n.load(nd); 217 externalIdMap.put(nd.getPrimitiveId(), n); 218 while (true) { 219 int event = parser.next(); 220 if (event == XMLStreamConstants.START_ELEMENT) { 221 if ("tag".equals(parser.getLocalName())) { 222 parseTag(n); 223 } else { 224 parseUnknown(); 225 } 226 } else if (event == XMLStreamConstants.END_ELEMENT) 227 return n; 228 } 229 } 230 231 protected Way parseWay() throws XMLStreamException { 232 WayData wd = new WayData(); 233 readCommon(wd); 234 Way w = new Way(wd.getId(), wd.getVersion()); 235 w.setVisible(wd.isVisible()); 236 w.load(wd); 237 externalIdMap.put(wd.getPrimitiveId(), w); 238 239 Collection<Long> nodeIds = new ArrayList<>(); 240 while (true) { 241 int event = parser.next(); 242 if (event == XMLStreamConstants.START_ELEMENT) { 243 switch (parser.getLocalName()) { 244 case "nd": 245 nodeIds.add(parseWayNode(w)); 246 break; 247 case "tag": 248 parseTag(w); 249 break; 250 default: 251 parseUnknown(); 252 } 253 } else if (event == XMLStreamConstants.END_ELEMENT) { 254 break; 255 } 256 } 257 if (w.isDeleted() && !nodeIds.isEmpty()) { 258 Main.info(tr("Deleted way {0} contains nodes", w.getUniqueId())); 259 nodeIds = new ArrayList<>(); 260 } 261 ways.put(wd.getUniqueId(), nodeIds); 262 return w; 263 } 264 265 private long parseWayNode(Way w) throws XMLStreamException { 266 if (parser.getAttributeValue(null, "ref") == null) { 267 throwException( 268 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", w.getUniqueId()) 269 ); 270 } 271 long id = getLong("ref"); 272 if (id == 0) { 273 throwException( 274 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id) 275 ); 276 } 277 jumpToEnd(); 278 return id; 279 } 280 281 protected Relation parseRelation() throws XMLStreamException { 282 RelationData rd = new RelationData(); 283 readCommon(rd); 284 Relation r = new Relation(rd.getId(), rd.getVersion()); 285 r.setVisible(rd.isVisible()); 286 r.load(rd); 287 externalIdMap.put(rd.getPrimitiveId(), r); 288 289 Collection<RelationMemberData> members = new ArrayList<>(); 290 while (true) { 291 int event = parser.next(); 292 if (event == XMLStreamConstants.START_ELEMENT) { 293 switch (parser.getLocalName()) { 294 case "member": 295 members.add(parseRelationMember(r)); 296 break; 297 case "tag": 298 parseTag(r); 299 break; 300 default: 301 parseUnknown(); 302 } 303 } else if (event == XMLStreamConstants.END_ELEMENT) { 304 break; 305 } 306 } 307 if (r.isDeleted() && !members.isEmpty()) { 308 Main.info(tr("Deleted relation {0} contains members", r.getUniqueId())); 309 members = new ArrayList<>(); 310 } 311 relations.put(rd.getUniqueId(), members); 312 return r; 313 } 314 315 private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException { 316 String role = null; 317 OsmPrimitiveType type = null; 318 long id = 0; 319 String value = parser.getAttributeValue(null, "ref"); 320 if (value == null) { 321 throwException(tr("Missing attribute ''ref'' on member in relation {0}.", r.getUniqueId())); 322 } 323 try { 324 id = Long.parseLong(value); 325 } catch (NumberFormatException e) { 326 throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(r.getUniqueId()), 327 value), e); 328 } 329 value = parser.getAttributeValue(null, "type"); 330 if (value == null) { 331 throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(id), Long.toString(r.getUniqueId()))); 332 } 333 try { 334 type = OsmPrimitiveType.fromApiTypeName(value); 335 } catch (IllegalArgumentException e) { 336 throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.", 337 Long.toString(id), Long.toString(r.getUniqueId()), value), e); 338 } 339 value = parser.getAttributeValue(null, "role"); 340 role = value; 341 342 if (id == 0) { 343 throwException(tr("Incomplete <member> specification with ref=0")); 344 } 345 jumpToEnd(); 346 return new RelationMemberData(role, type, id); 347 } 348 349 private void parseChangeset(Long uploadChangesetId) throws XMLStreamException { 350 351 Long id = null; 352 if (parser.getAttributeValue(null, "id") != null) { 353 id = getLong("id"); 354 } 355 // Read changeset info if neither upload-changeset nor id are set, or if they are both set to the same value 356 if (id == uploadChangesetId || (id != null && id.equals(uploadChangesetId))) { 357 uploadChangeset = new Changeset(id != null ? id.intValue() : 0); 358 while (true) { 359 int event = parser.next(); 360 if (event == XMLStreamConstants.START_ELEMENT) { 361 if ("tag".equals(parser.getLocalName())) { 362 parseTag(uploadChangeset); 363 } else { 364 parseUnknown(); 365 } 366 } else if (event == XMLStreamConstants.END_ELEMENT) 367 return; 368 } 369 } else { 370 jumpToEnd(false); 371 } 372 } 373 374 private void parseTag(Tagged t) throws XMLStreamException { 375 String key = parser.getAttributeValue(null, "k"); 376 String value = parser.getAttributeValue(null, "v"); 377 if (key == null || value == null) { 378 throwException(tr("Missing key or value attribute in tag.")); 379 } else { 380 t.put(key.intern(), value.intern()); 381 } 382 jumpToEnd(); 383 } 384 385 protected void parseUnknown(boolean printWarning) throws XMLStreamException { 386 final String element = parser.getLocalName(); 387 if (printWarning && ("note".equals(element) || "meta".equals(element))) { 388 // we know that Overpass API returns those elements 389 Main.debug(tr("Undefined element ''{0}'' found in input stream. Skipping.", element)); 390 } else if (printWarning) { 391 Main.info(tr("Undefined element ''{0}'' found in input stream. Skipping.", element)); 392 } 393 while (true) { 394 int event = parser.next(); 395 if (event == XMLStreamConstants.START_ELEMENT) { 396 parseUnknown(false); /* no more warning for inner elements */ 397 } else if (event == XMLStreamConstants.END_ELEMENT) 398 return; 399 } 400 } 401 402 protected void parseUnknown() throws XMLStreamException { 403 parseUnknown(true); 404 } 405 406 /** 407 * When cursor is at the start of an element, moves it to the end tag of that element. 408 * Nested content is skipped. 409 * 410 * This is basically the same code as parseUnknown(), except for the warnings, which 411 * are displayed for inner elements and not at top level. 412 * @param printWarning if {@code true}, a warning message will be printed if an unknown element is met 413 * @throws XMLStreamException if there is an error processing the underlying XML source 414 */ 415 private void jumpToEnd(boolean printWarning) throws XMLStreamException { 416 while (true) { 417 int event = parser.next(); 418 if (event == XMLStreamConstants.START_ELEMENT) { 419 parseUnknown(printWarning); 420 } else if (event == XMLStreamConstants.END_ELEMENT) 421 return; 422 } 423 } 424 425 private void jumpToEnd() throws XMLStreamException { 426 jumpToEnd(true); 427 } 428 429 private User createUser(String uid, String name) throws XMLStreamException { 430 if (uid == null) { 431 if (name == null) 432 return null; 433 return User.createLocalUser(name); 434 } 435 try { 436 long id = Long.parseLong(uid); 437 return User.createOsmUser(id, name); 438 } catch (NumberFormatException e) { 439 throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid), e); 440 } 441 return null; 442 } 443 444 /** 445 * Read out the common attributes and put them into current OsmPrimitive. 446 * @param current primitive to update 447 * @throws XMLStreamException if there is an error processing the underlying XML source 448 */ 449 private void readCommon(PrimitiveData current) throws XMLStreamException { 450 current.setId(getLong("id")); 451 if (current.getUniqueId() == 0) { 452 throwException(tr("Illegal object with ID=0.")); 453 } 454 455 String time = parser.getAttributeValue(null, "timestamp"); 456 if (time != null && !time.isEmpty()) { 457 current.setRawTimestamp((int) (DateUtils.tsFromString(time)/1000)); 458 } 459 460 String user = parser.getAttributeValue(null, "user"); 461 String uid = parser.getAttributeValue(null, "uid"); 462 current.setUser(createUser(uid, user)); 463 464 String visible = parser.getAttributeValue(null, "visible"); 465 if (visible != null) { 466 current.setVisible(Boolean.parseBoolean(visible)); 467 } 468 469 String versionString = parser.getAttributeValue(null, "version"); 470 int version = 0; 471 if (versionString != null) { 472 try { 473 version = Integer.parseInt(versionString); 474 } catch (NumberFormatException e) { 475 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", 476 Long.toString(current.getUniqueId()), versionString), e); 477 } 478 switch (ds.getVersion()) { 479 case "0.6": 480 if (version <= 0 && !current.isNew()) { 481 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", 482 Long.toString(current.getUniqueId()), versionString)); 483 } else if (version < 0 && current.isNew()) { 484 Main.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", 485 current.getUniqueId(), version, 0, "0.6")); 486 version = 0; 487 } 488 break; 489 default: 490 // should not happen. API version has been checked before 491 throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion())); 492 } 493 } else { 494 // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6 495 if (!current.isNew() && ds.getVersion() != null && "0.6".equals(ds.getVersion())) { 496 throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId()))); 497 } 498 } 499 current.setVersion(version); 500 501 String action = parser.getAttributeValue(null, "action"); 502 if (action == null) { 503 // do nothing 504 } else if ("delete".equals(action)) { 505 current.setDeleted(true); 506 current.setModified(current.isVisible()); 507 } else if ("modify".equals(action)) { 508 current.setModified(true); 509 } 510 511 String v = parser.getAttributeValue(null, "changeset"); 512 if (v == null) { 513 current.setChangesetId(0); 514 } else { 515 try { 516 current.setChangesetId(Integer.parseInt(v)); 517 } catch (IllegalArgumentException e) { 518 Main.debug(e.getMessage()); 519 if (current.isNew()) { 520 // for a new primitive we just log a warning 521 Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", 522 v, current.getUniqueId())); 523 current.setChangesetId(0); 524 } else { 525 // for an existing primitive this is a problem 526 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v), e); 527 } 528 } catch (IllegalStateException e) { 529 // thrown for positive changeset id on new primitives 530 Main.info(e.getMessage()); 531 current.setChangesetId(0); 532 } 533 if (current.getChangesetId() <= 0) { 534 if (current.isNew()) { 535 // for a new primitive we just log a warning 536 Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", 537 v, current.getUniqueId())); 538 current.setChangesetId(0); 539 } else { 540 // for an existing primitive this is a problem 541 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v)); 542 } 543 } 544 } 545 } 546 547 private long getLong(String name) throws XMLStreamException { 548 String value = parser.getAttributeValue(null, name); 549 if (value == null) { 550 throwException(tr("Missing required attribute ''{0}''.", name)); 551 } 552 try { 553 return Long.parseLong(value); 554 } catch (NumberFormatException e) { 555 throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.", name, value), e); 556 } 557 return 0; // should not happen 558 } 559 560 private static class OsmParsingException extends XMLStreamException { 561 562 OsmParsingException(String msg, Location location) { 563 super(msg); /* cannot use super(msg, location) because it messes with the message preventing localization */ 564 this.location = location; 565 } 566 567 OsmParsingException(String msg, Location location, Throwable th) { 568 super(msg, th); 569 this.location = location; 570 } 571 572 @Override 573 public String getMessage() { 574 String msg = super.getMessage(); 575 if (msg == null) { 576 msg = getClass().getName(); 577 } 578 if (getLocation() == null) 579 return msg; 580 msg += ' ' + tr("(at line {0}, column {1})", getLocation().getLineNumber(), getLocation().getColumnNumber()); 581 int offset = getLocation().getCharacterOffset(); 582 if (offset > -1) { 583 msg += ". "+ tr("{0} bytes have been read", offset); 584 } 585 return msg; 586 } 587 } 588 589 /** 590 * Exception thrown after user cancelation. 591 */ 592 private static final class OsmParsingCanceledException extends OsmParsingException implements ImportCancelException { 593 /** 594 * Constructs a new {@code OsmParsingCanceledException}. 595 * @param msg The error message 596 * @param location The parser location 597 */ 598 OsmParsingCanceledException(String msg, Location location) { 599 super(msg, location); 600 } 601 } 602 603 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 604 if (progressMonitor == null) { 605 progressMonitor = NullProgressMonitor.INSTANCE; 606 } 607 ProgressMonitor.CancelListener cancelListener = new ProgressMonitor.CancelListener() { 608 @Override public void operationCanceled() { 609 cancel = true; 610 } 611 }; 612 progressMonitor.addCancelListener(cancelListener); 613 CheckParameterUtil.ensureParameterNotNull(source, "source"); 614 try { 615 progressMonitor.beginTask(tr("Prepare OSM data...", 2)); 616 progressMonitor.indeterminateSubTask(tr("Parsing OSM data...")); 617 618 try (InputStreamReader ir = UTFInputStreamReader.create(source)) { 619 XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(ir); 620 setParser(parser); 621 parse(); 622 } 623 progressMonitor.worked(1); 624 625 progressMonitor.indeterminateSubTask(tr("Preparing data set...")); 626 prepareDataSet(); 627 progressMonitor.worked(1); 628 629 // iterate over registered postprocessors and give them each a chance 630 // to modify the dataset we have just loaded. 631 if (postprocessors != null) { 632 for (OsmServerReadPostprocessor pp : postprocessors) { 633 pp.postprocessDataSet(getDataSet(), progressMonitor); 634 } 635 } 636 return getDataSet(); 637 } catch (IllegalDataException e) { 638 throw e; 639 } catch (OsmParsingException e) { 640 throw new IllegalDataException(e.getMessage(), e); 641 } catch (XMLStreamException e) { 642 String msg = e.getMessage(); 643 Pattern p = Pattern.compile("Message: (.+)"); 644 Matcher m = p.matcher(msg); 645 if (m.find()) { 646 msg = m.group(1); 647 } 648 if (e.getLocation() != null) 649 throw new IllegalDataException(tr("Line {0} column {1}: ", 650 e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e); 651 else 652 throw new IllegalDataException(msg, e); 653 } catch (Exception e) { 654 throw new IllegalDataException(e); 655 } finally { 656 progressMonitor.finishTask(); 657 progressMonitor.removeCancelListener(cancelListener); 658 } 659 } 660 661 /** 662 * Parse the given input source and return the dataset. 663 * 664 * @param source the source input stream. Must not be null. 665 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed 666 * 667 * @return the dataset with the parsed data 668 * @throws IllegalDataException if an error was found while parsing the data from the source 669 * @throws IllegalArgumentException if source is null 670 */ 671 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 672 return new OsmReader().doParseDataSet(source, progressMonitor); 673 } 674}