/*
 * Copyright 2014 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.openrtb.json;

import static com.google.openrtb.json.OpenRtbJsonUtils.endArray;
import static com.google.openrtb.json.OpenRtbJsonUtils.endObject;
import static com.google.openrtb.json.OpenRtbJsonUtils.getCurrentName;
import static com.google.openrtb.json.OpenRtbJsonUtils.peekStructStart;
import static com.google.openrtb.json.OpenRtbJsonUtils.readCsvString;
import static com.google.openrtb.json.OpenRtbJsonUtils.startArray;
import static com.google.openrtb.json.OpenRtbJsonUtils.startObject;

import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.google.common.io.CharSource;
import com.google.common.io.Closeables;
import com.google.openrtb.Gender;
import com.google.openrtb.OpenRtb.APIFramework;
import com.google.openrtb.OpenRtb.AdPosition;
import com.google.openrtb.OpenRtb.AuctionType;
import com.google.openrtb.OpenRtb.BannerAdType;
import com.google.openrtb.OpenRtb.BidRequest;
import com.google.openrtb.OpenRtb.BidRequest.App;
import com.google.openrtb.OpenRtb.BidRequest.Content;
import com.google.openrtb.OpenRtb.BidRequest.Data;
import com.google.openrtb.OpenRtb.BidRequest.Data.Segment;
import com.google.openrtb.OpenRtb.BidRequest.Device;
import com.google.openrtb.OpenRtb.BidRequest.Geo;
import com.google.openrtb.OpenRtb.BidRequest.Imp;
import com.google.openrtb.OpenRtb.BidRequest.Imp.Audio;
import com.google.openrtb.OpenRtb.BidRequest.Imp.Banner;
import com.google.openrtb.OpenRtb.BidRequest.Imp.Banner.Format;
import com.google.openrtb.OpenRtb.BidRequest.Imp.Metric;
import com.google.openrtb.OpenRtb.BidRequest.Imp.Native;
import com.google.openrtb.OpenRtb.BidRequest.Imp.Pmp;
import com.google.openrtb.OpenRtb.BidRequest.Imp.Pmp.Deal;
import com.google.openrtb.OpenRtb.BidRequest.Imp.Video;
import com.google.openrtb.OpenRtb.BidRequest.Imp.Video.CompanionAd;
import com.google.openrtb.OpenRtb.BidRequest.Producer;
import com.google.openrtb.OpenRtb.BidRequest.Publisher;
import com.google.openrtb.OpenRtb.BidRequest.Regs;
import com.google.openrtb.OpenRtb.BidRequest.Site;
import com.google.openrtb.OpenRtb.BidRequest.Source;
import com.google.openrtb.OpenRtb.BidRequest.User;
import com.google.openrtb.OpenRtb.BidResponse;
import com.google.openrtb.OpenRtb.BidResponse.SeatBid;
import com.google.openrtb.OpenRtb.BidResponse.SeatBid.Bid;
import com.google.openrtb.OpenRtb.CompanionType;
import com.google.openrtb.OpenRtb.ConnectionType;
import com.google.openrtb.OpenRtb.ContentContext;
import com.google.openrtb.OpenRtb.ContentDeliveryMethod;
import com.google.openrtb.OpenRtb.CreativeAttribute;
import com.google.openrtb.OpenRtb.DeviceType;
import com.google.openrtb.OpenRtb.ExpandableDirection;
import com.google.openrtb.OpenRtb.FeedType;
import com.google.openrtb.OpenRtb.LocationService;
import com.google.openrtb.OpenRtb.LocationType;
import com.google.openrtb.OpenRtb.NoBidReason;
import com.google.openrtb.OpenRtb.PlaybackCessationMode;
import com.google.openrtb.OpenRtb.PlaybackMethod;
import com.google.openrtb.OpenRtb.ProductionQuality;
import com.google.openrtb.OpenRtb.Protocol;
import com.google.openrtb.OpenRtb.QAGMediaRating;
import com.google.openrtb.OpenRtb.VideoLinearity;
import com.google.openrtb.OpenRtb.VideoPlacementType;
import com.google.openrtb.OpenRtb.VolumeNormalizationMode;
import com.google.openrtb.util.ProtoUtils;
import com.google.protobuf.ByteString;
import java.io.CharArrayReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;

/**
 * Desserializes OpenRTB {@link BidRequest}/{@link BidResponse} messages from JSON.
 *
 * <p>This class is threadsafe.
 */
public class OpenRtbJsonReader extends AbstractOpenRtbJsonReader {

  protected OpenRtbJsonReader(OpenRtbJsonFactory factory) {
    super(factory);
  }

  /**
   * Desserializes a {@link BidRequest} from a JSON string, provided as a {@link ByteString}.
   */
  public BidRequest readBidRequest(ByteString bs) throws IOException {
    return readBidRequest(bs.newInput());
  }

  /**
   * Desserializes a {@link BidRequest} from a JSON string, provided as a {@link CharSequence}.
   */
  public BidRequest readBidRequest(CharSequence chars) throws IOException {
    return readBidRequest(CharSource.wrap(chars).openStream());
  }

  /**
   * Desserializes a {@link BidRequest} from JSON, streamed from a {@link Reader}.
   */
  public BidRequest readBidRequest(Reader reader) throws IOException {
    return ProtoUtils.built(readBidRequest(factory().getJsonFactory().createParser(reader)));
  }

  /**
   * Desserializes a {@link BidRequest} from JSON, streamed from an {@link InputStream}.
   */
  public BidRequest readBidRequest(InputStream is) throws IOException {
    try {
      return ProtoUtils.built(readBidRequest(factory().getJsonFactory().createParser(is)));
    } finally {
      Closeables.closeQuietly(is);
    }
  }

  /**
   * Desserializes a {@link BidRequest} from JSON, with a provided {@link JsonParser}
   * which allows several choices of input and encoding.
   */
  public final BidRequest.Builder readBidRequest(JsonParser par) throws IOException {
    if (emptyToNull(par)) {
      return null;
    }

    BidRequest.Builder req = BidRequest.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readBidRequestField(par, req, fieldName);
      }
    }
    return req;
  }

  protected void readBidRequestField(JsonParser par, BidRequest.Builder req, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "id":
        req.setId(par.getText());
        break;
      case "imp":
        for (startArray(par); endArray(par); par.nextToken()) {
          req.addImp(readImp(par));
        }
        break;
      case "site":
        req.setSite(readSite(par));
        break;
      case "app":
        req.setApp(readApp(par));
        break;
      case "device":
        req.setDevice(readDevice(par));
        break;
      case "user":
        req.setUser(readUser(par));
        break;
      case "test":
        req.setTest(par.getValueAsBoolean());
        break;
      case "at": {
          AuctionType value = AuctionType.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            req.setAt(value);
          }
        }
        break;
      case "tmax":
        req.setTmax(par.getIntValue());
        break;
      case "wseat":
        for (startArray(par); endArray(par); par.nextToken()) {
          req.addWseat(par.getText());
        }
        break;
      case "allimps":
        req.setAllimps(par.getValueAsBoolean());
        break;
      case "cur":
        for (startArray(par); endArray(par); par.nextToken()) {
          req.addCur(par.getText());
        }
        break;
      case "bcat":
        for (startArray(par); endArray(par); par.nextToken()) {
          String cat = par.getText();
          if (checkContentCategory(cat)) {
            req.addBcat(cat);
          }
        }
        break;
      case "badv":
        for (startArray(par); endArray(par); par.nextToken()) {
          req.addBadv(par.getText());
        }
        break;
      case "regs":
        req.setRegs(readRegs(par));
        break;
      case "bapp":
        for (startArray(par); endArray(par); par.nextToken()) {
          req.addBapp(par.getText());
        }
        break;
      case "bseat":
        for (startArray(par); endArray(par); par.nextToken()) {
          req.addBseat(par.getText());
        }
        break;
      case "wlang":
        for (startArray(par); endArray(par); par.nextToken()) {
          req.addWlang(par.getText());
        }
        break;
      case "source":
        req.setSource(readSource(par));
        break;
      default:
        readOther(req, par, fieldName);
    }
  }

  public final Source.Builder readSource(JsonParser par) throws IOException {
    Source.Builder source = Source.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readSourceField(par, source, fieldName);
      }
    }
    return source;
  }

  protected void readSourceField(JsonParser par, Source.Builder source, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "fd":
        source.setFd(par.getValueAsBoolean());
        break;
      case "tid":
        source.setTid(par.getText());
        break;
      case "pchain":
        source.setPchain(par.getText());
        break;
      default:
        readOther(source, par, fieldName);
    }
  }

  public final Regs.Builder readRegs(JsonParser par) throws IOException {
    Regs.Builder reg = Regs.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readRegsField(par, reg, fieldName);
      }
    }
    return reg;
  }

  protected void readRegsField(JsonParser par, Regs.Builder reg, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "coppa":
        reg.setCoppa(par.getValueAsBoolean());
        break;
      default:
        readOther(reg, par, fieldName);
    }
  }

  public final Imp.Builder readImp(JsonParser par) throws IOException {
    Imp.Builder imp = Imp.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readImpField(par, imp, fieldName);
      }
    }
    return imp;
  }

  protected void readImpField(JsonParser par, Imp.Builder imp, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "id":
        imp.setId(par.getText());
        break;
      case "banner":
        imp.setBanner(readBanner(par));
        break;
      case "video":
        imp.setVideo(readVideo(par));
        break;
      case "audio":
        imp.setAudio(readAudio(par));
        break;
      case "native":
        imp.setNative(readNative(par));
        break;
      case "displaymanager":
        imp.setDisplaymanager(par.getText());
        break;
      case "displaymanagerver":
        imp.setDisplaymanagerver(par.getText());
        break;
      case "instl":
        imp.setInstl(par.getValueAsBoolean());
        break;
      case "tagid":
        imp.setTagid(par.getText());
        break;
      case "bidfloor":
        imp.setBidfloor(par.getValueAsDouble());
        break;
      case "bidfloorcur":
        imp.setBidfloorcur(par.getText());
        break;
      case "secure":
        imp.setSecure(par.getValueAsBoolean());
        break;
      case "iframebuster":
        for (startArray(par); endArray(par); par.nextToken()) {
          imp.addIframebuster(par.getText());
        }
        break;
      case "pmp":
        imp.setPmp(readPmp(par));
        break;
      case "clickbrowser":
        imp.setClickbrowser(par.getValueAsBoolean());
        break;
      case "exp":
        imp.setExp(par.getIntValue());
        break;
      case "metric":
        for (startArray(par); endArray(par); par.nextToken()) {
          imp.addMetric(readMetric(par));
        }
        break;
      default:
        readOther(imp, par, fieldName);
    }
  }

  public final Metric.Builder readMetric(JsonParser par) throws IOException {
    Metric.Builder metric = Metric.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readMetricField(par, metric, fieldName);
      }
    }
    return metric;
  }

  protected void readMetricField(JsonParser par, Metric.Builder metric, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "type":
        metric.setType(par.getText());
        break;
      case "value":
        metric.setValue(par.getValueAsDouble());
        break;
      case "vendor":
        metric.setVendor(par.getText());
        break;
      default:
        readOther(metric, par, fieldName);
    }
  }

  public final Native.Builder readNative(JsonParser par) throws IOException {
    Native.Builder nativ = Native.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readNativeField(par, nativ, fieldName);
      }
    }
    return nativ;
  }

  protected void readNativeField(JsonParser par, Native.Builder nativ, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "request":
        if (par.getCurrentToken() == JsonToken.VALUE_STRING) {
          nativ.setRequestNative(factory().newNativeReader().readNativeRequest(new CharArrayReader(
              par.getTextCharacters(), par.getTextOffset(), par.getTextLength())));
        } else { // Object
          nativ.setRequestNative(factory().newNativeReader().readNativeRequest(par));
        }
        break;
      case "ver":
        nativ.setVer(par.getText());
        break;
      case "api":
        for (startArray(par); endArray(par); par.nextToken()) {
          APIFramework value = APIFramework.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            nativ.addApi(value);
          }
        }
        break;
      case "battr":
        for (startArray(par); endArray(par); par.nextToken()) {
          CreativeAttribute value = CreativeAttribute.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            nativ.addBattr(value);
          }
        }
        break;
      default:
        readOther(nativ, par, fieldName);
    }
  }

  public final Pmp.Builder readPmp(JsonParser par) throws IOException {
    Pmp.Builder pmp = Pmp.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readPmpField(par, pmp, fieldName);
      }
    }
    return pmp;
  }

  protected void readPmpField(JsonParser par, Pmp.Builder pmp, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "private_auction":
        pmp.setPrivateAuction(par.getValueAsBoolean());
        break;
      case "deals":
        for (startArray(par); endArray(par); par.nextToken()) {
          pmp.addDeals(readDeal(par));
        }
        break;
      default:
        readOther(pmp, par, fieldName);
    }
  }

  public final Deal.Builder readDeal(JsonParser par) throws IOException {
    Deal.Builder deal = Deal.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readDealField(par, deal, fieldName);
      }
    }
    return deal;
  }

  protected void readDealField(JsonParser par, Deal.Builder deal, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "id":
        deal.setId(par.getText());
        break;
      case "bidfloor":
        deal.setBidfloor(par.getValueAsDouble());
        break;
      case "bidfloorcur":
        deal.setBidfloorcur(par.getText());
        break;
      case "wseat":
        for (startArray(par); endArray(par); par.nextToken()) {
          deal.addWseat(par.getText());
        }
        break;
      case "wadomain":
        for (startArray(par); endArray(par); par.nextToken()) {
          deal.addWadomain(par.getText());
        }
        break;
      case "at": {
          AuctionType value = AuctionType.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            deal.setAt(value);
          }
        }
        break;
      default:
        readOther(deal, par, fieldName);
    }
  }

  public final Video.Builder readVideo(JsonParser par) throws IOException {
    Video.Builder video = Video.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readVideoField(par, video, fieldName);
      }
    }
    return video;
  }

  @SuppressWarnings("deprecation")
  protected void readVideoField(JsonParser par, Video.Builder video, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "mimes":
        for (startArray(par); endArray(par); par.nextToken()) {
          video.addMimes(par.getText());
        }
        break;
      case "minduration":
        video.setMinduration(par.getIntValue());
        break;
      case "maxduration":
        video.setMaxduration(par.getIntValue());
        break;
      case "protocol": {
          Protocol value = Protocol.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            video.setProtocol(value);
          }
        }
        break;
      case "protocols":
        for (startArray(par); endArray(par); par.nextToken()) {
          Protocol value = Protocol.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            video.addProtocols(value);
          }
        }
        break;
      case "w":
        video.setW(par.getIntValue());
        break;
      case "h":
        video.setH(par.getIntValue());
        break;
      case "startdelay":
        video.setStartdelay(par.getIntValue());
        break;
      case "linearity": {
          VideoLinearity value = VideoLinearity.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            video.setLinearity(value);
          }
        }
        break;
      case "sequence":
        video.setSequence(par.getIntValue());
        break;
      case "battr":
        for (startArray(par); endArray(par); par.nextToken()) {
          CreativeAttribute value = CreativeAttribute.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            video.addBattr(value);
          }
        }
        break;
      case "maxextended":
        video.setMaxextended(par.getIntValue());
        break;
      case "minbitrate":
        video.setMinbitrate(par.getIntValue());
        break;
      case "maxbitrate":
        video.setMaxbitrate(par.getIntValue());
        break;
      case "boxingallowed":
        video.setBoxingallowed(par.getValueAsBoolean());
        break;
      case "playbackmethod":
        for (startArray(par); endArray(par); par.nextToken()) {
          PlaybackMethod value = PlaybackMethod.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            video.addPlaybackmethod(value);
          }
        }
        break;
      case "delivery":
        for (startArray(par); endArray(par); par.nextToken()) {
          ContentDeliveryMethod value = ContentDeliveryMethod.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            video.addDelivery(value);
          }
        }
        break;
      case "pos": {
          AdPosition value = AdPosition.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            video.setPos(value);
          }
        }
        break;
      case "companionad":
        if (peekStructStart(par) == JsonToken.START_ARRAY) {
          // OpenRTB 2.2+
          for (startArray(par); endArray(par); par.nextToken()) {
            video.addCompanionad(readBanner(par));
          }
        } else { // START_OBJECT
          // OpenRTB 2.1-
          video.setCompanionad21(readCompanionAd(par));
        }
        break;
      case "api":
        for (startArray(par); endArray(par); par.nextToken()) {
          APIFramework value = APIFramework.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            video.addApi(value);
          }
        }
        break;
      case "companiontype":
        for (startArray(par); endArray(par); par.nextToken()) {
          CompanionType value = CompanionType.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            video.addCompaniontype(value);
          }
        }
        break;
      case "skip":
        video.setSkip(par.getValueAsBoolean());
        break;
      case "skipmin":
        video.setSkipmin(par.getIntValue());
        break;
      case "skipafter":
        video.setSkipafter(par.getIntValue());
        break;
      case "placement":
        video.setPlacement(VideoPlacementType.forNumber(par.getIntValue()));
        break;
      case "playbackend":
        video.setPlaybackend(PlaybackCessationMode.forNumber(par.getIntValue()));
        break;
      default:
        readOther(video, par, fieldName);
    }
  }

  public final CompanionAd.Builder readCompanionAd(JsonParser par) throws IOException {
    CompanionAd.Builder companionad = CompanionAd.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readCompanionAdField(par, companionad, fieldName);
      }
    }
    return companionad;
  }

  protected void readCompanionAdField(
      JsonParser par, CompanionAd.Builder companionad, String fieldName) throws IOException {
    switch (fieldName) {
      case "banner":
        for (startArray(par); endArray(par); par.nextToken()) {
          companionad.addBanner(readBanner(par));
        }
        break;
      default:
        readOther(companionad, par, fieldName);
    }
  }

  public final Audio.Builder readAudio(JsonParser par) throws IOException {
    Audio.Builder audio = Audio.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readAudioField(par, audio, fieldName);
      }
    }
    return audio;
  }

  protected void readAudioField(JsonParser par, Audio.Builder audio, String fieldName)
      throws IOException {
    // Video & Audio common

    switch (fieldName) {
      case "mimes":
        for (startArray(par); endArray(par); par.nextToken()) {
          audio.addMimes(par.getText());
        }
        break;
      case "minduration":
        audio.setMinduration(par.getIntValue());
        break;
      case "maxduration":
        audio.setMaxduration(par.getIntValue());
        break;
      case "protocols":
        for (startArray(par); endArray(par); par.nextToken()) {
          Protocol value = Protocol.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            audio.addProtocols(value);
          }
        }
        break;
      case "startdelay":
        audio.setStartdelay(par.getIntValue());
        break;
      case "sequence":
        audio.setSequence(par.getIntValue());
        break;
      case "battr":
        for (startArray(par); endArray(par); par.nextToken()) {
          CreativeAttribute value = CreativeAttribute.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            audio.addBattr(value);
          }
        }
        break;
      case "maxextended":
        audio.setMaxextended(par.getIntValue());
        break;
      case "minbitrate":
        audio.setMinbitrate(par.getIntValue());
        break;
      case "maxbitrate":
        audio.setMaxbitrate(par.getIntValue());
        break;
      case "delivery":
        for (startArray(par); endArray(par); par.nextToken()) {
          ContentDeliveryMethod value = ContentDeliveryMethod.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            audio.addDelivery(value);
          }
        }
        break;
      case "companionad":
        if (peekStructStart(par) == JsonToken.START_ARRAY) {
          // OpenRTB 2.2+
          for (startArray(par); endArray(par); par.nextToken()) {
            audio.addCompanionad(readBanner(par));
          }
        }
        break;
      case "api":
        for (startArray(par); endArray(par); par.nextToken()) {
          APIFramework value = APIFramework.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            audio.addApi(value);
          }
        }
        break;
      case "companiontype":
        for (startArray(par); endArray(par); par.nextToken()) {
          CompanionType value = CompanionType.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            audio.addCompaniontype(value);
          }
        }
        break;

      // Audio only

      case "maxseq":
        audio.setMaxseq(par.getIntValue());
        break;
      case "feed": {
          FeedType feed = FeedType.forNumber(par.getIntValue());
          if (checkEnum(feed)) {
            audio.setFeed(feed);
          }
        }
        break;
      case "stitched":
        audio.setStitched(par.getValueAsBoolean());
        break;
      case "nvol": {
          VolumeNormalizationMode value = VolumeNormalizationMode.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            audio.setNvol(value);
          }
        }
        break;

      default:
        readOther(audio, par, fieldName);
    }
  }

  public final Banner.Builder readBanner(JsonParser par) throws IOException {
    Banner.Builder banner = Banner.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readBannerField(par, banner, fieldName);
      }
    }
    return banner;
  }

  @SuppressWarnings("deprecation")
  protected void readBannerField(JsonParser par, Banner.Builder banner, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "w":
        banner.setW(par.getIntValue());
        break;
      case "h":
        banner.setH(par.getIntValue());
        break;
      case "wmax":
        banner.setWmax(par.getIntValue());
        break;
      case "hmax":
        banner.setHmax(par.getIntValue());
        break;
      case "wmin":
        banner.setWmin(par.getIntValue());
        break;
      case "hmin":
        banner.setHmin(par.getIntValue());
        break;
      case "id":
        banner.setId(par.getText());
        break;
      case "btype":
        for (startArray(par); endArray(par); par.nextToken()) {
          BannerAdType value = BannerAdType.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            banner.addBtype(value);
          }
        }
        break;
      case "battr":
        for (startArray(par); endArray(par); par.nextToken()) {
          CreativeAttribute value = CreativeAttribute.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            banner.addBattr(value);
          }
        }
        break;
      case "pos": {
          AdPosition value = AdPosition.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            banner.setPos(value);
          }
        }
        break;
      case "mimes":
        for (startArray(par); endArray(par); par.nextToken()) {
          banner.addMimes(par.getText());
        }
        break;
      case "topframe":
        banner.setTopframe(par.getValueAsBoolean());
        break;
      case "expdir":
        for (startArray(par); endArray(par); par.nextToken()) {
          ExpandableDirection value = ExpandableDirection.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            banner.addExpdir(value);
          }
        }
        break;
      case "api":
        for (startArray(par); endArray(par); par.nextToken()) {
          APIFramework value = APIFramework.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            banner.addApi(value);
          }
        }
        break;
      case "format":
        for (startArray(par); endArray(par); par.nextToken()) {
          banner.addFormat(readFormat(par));
        }
        break;
      case "vcm":
        banner.setVcm(par.getValueAsBoolean());
        break;
      default:
        readOther(banner, par, fieldName);
    }
  }

  public final Format.Builder readFormat(JsonParser par) throws IOException {
    Format.Builder format = Format.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readFormatField(par, format, fieldName);
      }
    }
    return format;
  }

  protected void readFormatField(JsonParser par, Format.Builder format, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "w":
        format.setW(par.getIntValue());
        break;
      case "h":
        format.setH(par.getIntValue());
        break;
      case "wratio":
        format.setWratio(par.getIntValue());
        break;
      case "hratio":
        format.setHratio(par.getIntValue());
        break;
      case "wmin":
        format.setWmin(par.getIntValue());
        break;
      default:
        readOther(format, par, fieldName);
    }
  }

  public final Site.Builder readSite(JsonParser par) throws IOException {
    Site.Builder site = Site.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readSiteField(par, site, fieldName);
      }
    }
    return site;
  }

  protected void readSiteField(JsonParser par, Site.Builder site, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "id":
        site.setId(par.getText());
        break;
      case "name":
        site.setName(par.getText());
        break;
      case "domain":
        site.setDomain(par.getText());
        break;
      case "cat":
        for (startArray(par); endArray(par); par.nextToken()) {
          String cat = par.getText();
          if (checkContentCategory(cat)) {
            site.addCat(cat);
          }
        }
        break;
      case "sectioncat":
        for (startArray(par); endArray(par); par.nextToken()) {
          String cat = par.getText();
          if (checkContentCategory(cat)) {
            site.addSectioncat(cat);
          }
        }
        break;
      case "pagecat":
        for (startArray(par); endArray(par); par.nextToken()) {
          String cat = par.getText();
          if (checkContentCategory(cat)) {
            site.addPagecat(cat);
          }
        }
        break;
      case "page":
        site.setPage(par.getText());
        break;
      case "ref":
        site.setRef(par.getText());
        break;
      case "search":
        site.setSearch(par.getText());
        break;
      case "mobile":
        site.setMobile(par.getValueAsBoolean());
        break;
      case "privacypolicy":
        site.setPrivacypolicy(par.getValueAsBoolean());
        break;
      case "publisher":
        site.setPublisher(readPublisher(par));
        break;
      case "content":
        site.setContent(readContent(par));
        break;
      case "keywords":
        site.setKeywords(readCsvString(par));
        break;
      default:
        readOther(site, par, fieldName);
    }
  }

  public final App.Builder readApp(JsonParser par) throws IOException {
    App.Builder app = App.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String name = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readAppField(par, app, name);
      }
    }
    return app;
  }

  protected void readAppField(JsonParser par, App.Builder app, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "id":
        app.setId(par.getText());
        break;
      case "name":
        app.setName(par.getText());
        break;
      case "bundle":
        app.setBundle(par.getText());
        break;
      case "domain":
        app.setDomain(par.getText());
        break;
      case "storeurl":
        app.setStoreurl(par.getText());
        break;
      case "cat":
        for (startArray(par); endArray(par); par.nextToken()) {
          String cat = par.getText();
          if (checkContentCategory(cat)) {
            app.addCat(cat);
          }
        }
        break;
      case "sectioncat":
        for (startArray(par); endArray(par); par.nextToken()) {
          String cat = par.getText();
          if (checkContentCategory(cat)) {
            app.addSectioncat(cat);
          }
        }
        break;
      case "pagecat":
        for (startArray(par); endArray(par); par.nextToken()) {
          String cat = par.getText();
          if (checkContentCategory(cat)) {
            app.addPagecat(cat);
          }
        }
        break;
      case "ver":
        app.setVer(par.getText());
        break;
      case "privacypolicy":
        app.setPrivacypolicy(par.getValueAsBoolean());
        break;
      case "paid":
        app.setPaid(par.getValueAsBoolean());
        break;
      case "publisher":
        app.setPublisher(readPublisher(par));
        break;
      case "content":
        app.setContent(readContent(par));
        break;
      case "keywords":
        app.setKeywords(readCsvString(par));
        break;
      default:
        readOther(app, par, fieldName);
    }
  }

  public final Content.Builder readContent(JsonParser par) throws IOException {
    Content.Builder content = Content.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String name = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readContentField(par, content, name);
      }
    }
    return content;
  }

  @SuppressWarnings("deprecation")
  protected void readContentField(JsonParser par, Content.Builder content, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "id":
        content.setId(par.getText());
        break;
      case "episode":
        content.setEpisode(par.getIntValue());
        break;
      case "title":
        content.setTitle(par.getText());
        break;
      case "series":
        content.setSeries(par.getText());
        break;
      case "season":
        content.setSeason(par.getText());
        break;
      case "producer":
        content.setProducer(readProducer(par));
        break;
      case "url":
        content.setUrl(par.getText());
        break;
      case "cat":
        for (startArray(par); endArray(par); par.nextToken()) {
          String cat = par.getText();
          if (checkContentCategory(cat)) {
            content.addCat(cat);
          }
        }
        break;
      case "videoquality": {
          ProductionQuality value = ProductionQuality.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            content.setVideoquality(value);
          }
        }
        break;
      case "context":
        try {
          // JsonParseException may be thrown because value is string in
          // 2.2 and earlier, this allows for backwards compatibility.
          ContentContext value = ContentContext.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            content.setContext(value);
          }
        } catch (JsonParseException jpe) {
        }
        break;
      case "contentrating":
        content.setContentrating(par.getText());
        break;
      case "userrating":
        content.setUserrating(par.getText());
        break;
      case "qagmediarating": {
          QAGMediaRating value = QAGMediaRating.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            content.setQagmediarating(value);
          }
        }
        break;
      case "keywords":
        content.setKeywords(readCsvString(par));
        break;
      case "livestream":
        content.setLivestream(par.getValueAsBoolean());
        break;
      case "sourcerelationship":
        content.setSourcerelationship(par.getValueAsBoolean());
        break;
      case "len":
        content.setLen(par.getIntValue());
        break;
      case "language":
        content.setLanguage(par.getText());
        break;
      case "embeddable":
        content.setEmbeddable(par.getValueAsBoolean());
        break;
      case "artist":
        content.setArtist(par.getText());
        break;
      case "genre":
        content.setGenre(par.getText());
        break;
      case "album":
        content.setAlbum(par.getText());
        break;
      case "isrc":
        content.setIsrc(par.getText());
        break;
      case "prodq": {
        ProductionQuality value = ProductionQuality.forNumber(par.getIntValue());
        if (checkEnum(value)) {
          content.setProdq(value);
        }
      }
      break;
      default:
        readOther(content, par, fieldName);
    }
  }

  public final Producer.Builder readProducer(JsonParser par) throws IOException {
    Producer.Builder producer = Producer.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readProducerField(par, producer, fieldName);
      }
    }
    return producer;
  }

  protected void readProducerField(JsonParser par, Producer.Builder producer, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "id":
        producer.setId(par.getText());
        break;
      case "name":
        producer.setName(par.getText());
        break;
      case "cat":
        for (startArray(par); endArray(par); par.nextToken()) {
          String cat = par.getText();
          if (checkContentCategory(cat)) {
            producer.addCat(cat);
          }
        }
        break;
      case "domain":
        producer.setDomain(par.getText());
        break;
      default:
        readOther(producer, par, fieldName);
    }
  }

  public final Publisher.Builder readPublisher(JsonParser par) throws IOException {
    Publisher.Builder publisher = Publisher.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readPublisherField(par, publisher, fieldName);
      }
    }
    return publisher;
  }

  protected void readPublisherField(JsonParser par, Publisher.Builder publisher, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "id":
        publisher.setId(par.getText());
        break;
      case "name":
        publisher.setName(par.getText());
        break;
      case "cat":
        for (startArray(par); endArray(par); par.nextToken()) {
          String cat = par.getText();
          if (checkContentCategory(cat)) {
            publisher.addCat(cat);
          }
        }
        break;
      case "domain":
        publisher.setDomain(par.getText());
        break;
      default:
        readOther(publisher, par, fieldName);
    }
  }

  public final Device.Builder readDevice(JsonParser par) throws IOException {
    Device.Builder device = Device.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readDeviceField(par, device, fieldName);
      }
    }
    return device;
  }

  protected void readDeviceField(JsonParser par, Device.Builder device, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "ua":
        device.setUa(par.getText());
        break;
      case "geo":
        device.setGeo(readGeo(par));
        break;
      case "dnt":
        device.setDnt(par.getValueAsBoolean());
        break;
      case "lmt":
        device.setLmt(par.getValueAsBoolean());
        break;
      case "ip":
        device.setIp(par.getText());
        break;
      case "ipv6":
        device.setIpv6(par.getText());
        break;
      case "devicetype": {
          DeviceType value = DeviceType.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            device.setDevicetype(value);
          }
        }
        break;
      case "make":
        device.setMake(par.getText());
        break;
      case "model":
        device.setModel(par.getText());
        break;
      case "os":
        device.setOs(par.getText());
        break;
      case "osv":
        device.setOsv(par.getText());
        break;
      case "hwv":
        device.setHwv(par.getText());
        break;
      case "w":
        device.setW(par.getIntValue());
        break;
      case "h":
        device.setH(par.getIntValue());
        break;
      case "ppi":
        device.setPpi(par.getIntValue());
        break;
      case "pxratio":
        device.setPxratio(par.getValueAsDouble());
        break;
      case "js":
        device.setJs(par.getValueAsBoolean());
        break;
      case "flashver":
        device.setFlashver(par.getText());
        break;
      case "language":
        device.setLanguage(par.getText());
        break;
      case "carrier":
        device.setCarrier(par.getText());
        break;
      case "connectiontype": {
          ConnectionType value = ConnectionType.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            device.setConnectiontype(value);
          }
        }
        break;
      case "ifa":
        device.setIfa(par.getText());
        break;
      case "didsha1":
        device.setDidsha1(par.getText());
        break;
      case "didmd5":
        device.setDidmd5(par.getText());
        break;
      case "dpidsha1":
        device.setDpidsha1(par.getText());
        break;
      case "dpidmd5":
        device.setDpidmd5(par.getText());
        break;
      case "macsha1":
        device.setMacsha1(par.getText());
        break;
      case "macmd5":
        device.setMacmd5(par.getText());
        break;
      case "geofetch":
        device.setGeofetch(par.getValueAsBoolean());
        break;
      case "mccmnc":
        device.setMccmnc(par.getText());
        break;
      default:
        readOther(device, par, fieldName);
    }
  }

  public final Geo.Builder readGeo(JsonParser par) throws IOException {
    Geo.Builder geo = Geo.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readGeoField(par, geo, fieldName);
      }
    }
    return geo;
  }

  protected void readGeoField(JsonParser par, Geo.Builder geo, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "lat":
        geo.setLat(par.getValueAsDouble());
        break;
      case "lon":
        geo.setLon(par.getValueAsDouble());
        break;
      case "type": {
          LocationType value = LocationType.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            geo.setType(value);
          }
        }
        break;
      case "country":
        geo.setCountry(par.getText());
        break;
      case "region":
        geo.setRegion(par.getText());
        break;
      case "regionfips104":
        geo.setRegionfips104(par.getText());
        break;
      case "metro":
        geo.setMetro(par.getText());
        break;
      case "city":
        geo.setCity(par.getText());
        break;
      case "zip":
        geo.setZip(par.getText());
        break;
      case "utcoffset":
        geo.setUtcoffset(par.getIntValue());
        break;
      case "accuracy":
        geo.setAccuracy(par.getIntValue());
        break;
      case "lastfix":
        geo.setLastfix(par.getIntValue());
        break;
      case "ipservice": {
          LocationService value = LocationService.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            geo.setIpservice(value);
          }
        }
        break;
      default:
        readOther(geo, par, fieldName);
    }
  }

  public final User.Builder readUser(JsonParser par) throws IOException {
    User.Builder user = User.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readUserField(par, user, fieldName);
      }
    }
    return user;
  }

  protected void readUserField(JsonParser par, User.Builder user, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "id":
        user.setId(par.getText());
        break;
      case "buyerid":  // Compatibility with broken 2.3 spec; fixed in 2.3.1
      case "buyeruid":
        user.setBuyeruid(par.getText());
        break;
      case "yob":
        user.setYob(par.getIntValue());
        break;
      case "gender": {
        Gender value = Gender.forCode(par.getText());
        if (checkEnum(value)) {
          user.setGender(value.code());
        }
        break;
      }
      case "keywords":
        user.setKeywords(readCsvString(par));
        break;
      case "customdata":
        user.setCustomdata(par.getText());
        break;
      case "geo":
        user.setGeo(readGeo(par));
        break;
      case "data":
        for (startArray(par); endArray(par); par.nextToken()) {
          user.addData(readData(par));
        }
        break;
      default:
        readOther(user, par, fieldName);
    }
  }

  public final Data.Builder readData(JsonParser par) throws IOException {
    Data.Builder data = Data.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readDataField(par, data, fieldName);
      }
    }
    return data;
  }

  protected void readDataField(JsonParser par, Data.Builder data, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "id":
        data.setId(par.getText());
        break;
      case "name":
        data.setName(par.getText());
        break;
      case "segment":
        for (startArray(par); endArray(par); par.nextToken()) {
          data.addSegment(readSegment(par));
        }
        break;
      default:
        readOther(data, par, fieldName);
    }
  }

  public final Segment.Builder readSegment(JsonParser par) throws IOException {
    Segment.Builder segment = Segment.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readSegmentField(par, segment, fieldName);
      }
    }
    return segment;
  }

  protected void readSegmentField(JsonParser par, Segment.Builder segment, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "id":
        segment.setId(par.getText());
        break;
      case "name":
        segment.setName(par.getText());
        break;
      case "value":
        segment.setValue(par.getText());
        break;
      default:
        readOther(segment, par, fieldName);
    }
  }

  /**
   * Desserializes a {@link BidResponse} from a JSON string, provided as a {@link ByteString}.
   */
  public BidResponse readBidResponse(ByteString bs) throws IOException {
    return readBidResponse(bs.newInput());
  }

  /**
   * Desserializes a {@link BidResponse} from a JSON string, provided as a {@link CharSequence}.
   */
  public BidResponse readBidResponse(CharSequence chars) throws IOException {
    return readBidResponse(CharSource.wrap(chars).openStream());
  }

  /**
   * Desserializes a {@link BidResponse} from JSON, streamed from a {@link Reader}.
   */
  public BidResponse readBidResponse(Reader reader) throws IOException {
    return ProtoUtils.built(readBidResponse(factory().getJsonFactory().createParser(reader)));
  }

  /**
   * Desserializes a {@link BidResponse} from JSON, streamed from an {@link InputStream}.
   */
  public BidResponse readBidResponse(InputStream is) throws IOException {
    try {
      return ProtoUtils.built(readBidResponse(factory().getJsonFactory().createParser(is)));
    } finally {
      Closeables.closeQuietly(is);
    }
  }

  /**
   * Desserializes a {@link BidResponse} from JSON, with a provided {@link JsonParser}
   * which allows several choices of input and encoding.
   */
  public final BidResponse.Builder readBidResponse(JsonParser par) throws IOException {
    if (emptyToNull(par)) {
      return null;
    }
    BidResponse.Builder resp = BidResponse.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readBidResponseField(par, resp, fieldName);
      }
    }
    return resp;
  }

  protected void readBidResponseField(JsonParser par, BidResponse.Builder resp, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "id":
        resp.setId(par.getText());
        break;
      case "seatbid":
        for (startArray(par); endArray(par); par.nextToken()) {
          resp.addSeatbid(readSeatBid(par));
        }
        break;
      case "bidid":
        resp.setBidid(par.getText());
        break;
      case "cur":
        resp.setCur(par.getText());
        break;
      case "customdata":
        resp.setCustomdata(par.getText());
        break;
      case "nbr": {
          NoBidReason value = NoBidReason.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            resp.setNbr(value);
          }
        }
        break;
      default:
        readOther(resp, par, fieldName);
    }
  }

  public final SeatBid.Builder readSeatBid(JsonParser par) throws IOException {
    SeatBid.Builder seatbid = SeatBid.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readSeatBidField(par, seatbid, fieldName);
      }
    }
    return seatbid;
  }

  protected void readSeatBidField(JsonParser par, SeatBid.Builder seatbid, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "bid":
        for (startArray(par); endArray(par); par.nextToken()) {
          seatbid.addBid(readBid(par));
        }
        break;
      case "seat":
        seatbid.setSeat(par.getText());
        break;
      case "group":
        seatbid.setGroup(par.getValueAsBoolean());
        break;
      default:
        readOther(seatbid, par, fieldName);
    }
  }

  public final Bid.Builder readBid(JsonParser par) throws IOException {
    Bid.Builder bid = Bid.newBuilder();
    for (startObject(par); endObject(par); par.nextToken()) {
      String fieldName = getCurrentName(par);
      if (par.nextToken() != JsonToken.VALUE_NULL) {
        readBidField(par, bid, fieldName);
      }
    }
    return bid;
  }

  protected void readBidField(JsonParser par, Bid.Builder bid, String fieldName)
      throws IOException {
    switch (fieldName) {
      case "id":
        bid.setId(par.getText());
        break;
      case "impid":
        bid.setImpid(par.getText());
        break;
      case "price":
        bid.setPrice(par.getValueAsDouble());
        break;
      case "adid":
        bid.setAdid(par.getText());
        break;
      case "nurl":
        bid.setNurl(par.getText());
        break;
      case "adm":
        if (par.getCurrentToken() == JsonToken.VALUE_STRING) {
          String valueString = par.getText();
          if (valueString.startsWith("{")) {
            bid.setAdmNative(factory().newNativeReader().readNativeResponse(valueString));
          } else {
            bid.setAdm(valueString);
          }
        } else { // Object
          bid.setAdmNative(factory().newNativeReader().readNativeResponse(par));
        }
        break;
      case "adomain":
        for (startArray(par); endArray(par); par.nextToken()) {
          bid.addAdomain(par.getText());
        }
        break;
      case "bundle":
        bid.setBundle(par.getText());
        break;
      case "iurl":
        bid.setIurl(par.getText());
        break;
      case "cid":
        bid.setCid(par.getText());
        break;
      case "crid":
        bid.setCrid(par.getText());
        break;
      case "cat":
        for (startArray(par); endArray(par); par.nextToken()) {
          String cat = par.getText();
          if (checkContentCategory(cat)) {
            bid.addCat(cat);
          }
        }
        break;
      case "attr":
        for (startArray(par); endArray(par); par.nextToken()) {
          CreativeAttribute value = CreativeAttribute.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            bid.addAttr(value);
          }
        }
        break;
      case "dealid":
        bid.setDealid(par.getText());
        break;
      case "w":
        bid.setW(par.getIntValue());
        break;
      case "h":
        bid.setH(par.getIntValue());
        break;
      case "api": {
          APIFramework value = APIFramework.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            bid.setApi(value);
          }
        }
        break;
      case "protocol": {
          Protocol value = Protocol.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            bid.setProtocol(value);
          }
        }
        break;
      case "qagmediarating": {
          QAGMediaRating value = QAGMediaRating.forNumber(par.getIntValue());
          if (checkEnum(value)) {
            bid.setQagmediarating(value);
          }
        }
        break;
      case "exp":
        bid.setExp(par.getIntValue());
        break;
      case "burl":
        bid.setBurl(par.getText());
        break;
      case "lurl":
        bid.setLurl(par.getText());
        break;
      case "tactic":
        bid.setTactic(par.getText());
        break;
      case "language":
        bid.setLanguage(par.getText());
        break;
      case "wratio":
        bid.setWratio(par.getIntValue());
        break;
      case "hratio":
        bid.setHratio(par.getIntValue());
        break;
      default:
        readOther(bid, par, fieldName);
    }
  }
}


/*
 * Copyright 2014 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.openrtb.util;

import static com.google.common.base.Preconditions.checkNotNull;

import com.google.common.collect.ImmutableList;
import com.google.protobuf.Descriptors.FieldDescriptor;
import com.google.protobuf.Message;
import com.google.protobuf.MessageLite;
import com.google.protobuf.MessageLiteOrBuilder;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.function.Predicate;
import javax.annotation.Nullable;

/**
 * Some conveniences for protoc-generated classes.
 */
public final class ProtoUtils {
  public static final Predicate<FieldDescriptor> NOT_EXTENSION = fd -> !fd.isExtension();

  private ProtoUtils() {
  }

  /**
   * Given a message-or-builder, returns a message, invoking the builder if necessary.
   */
  @SuppressWarnings("unchecked")
  public static <I extends MessageLiteOrBuilder, O extends MessageLite> O built(@Nullable I msg) {
    return msg instanceof MessageLite.Builder
        ? (O) ((MessageLite.Builder) msg).build()
        : (O) msg;
  }

  /**
   * Given a message-or-builder, return a builder, invoking toBuilder() if necessary.
   */
  @SuppressWarnings("unchecked")
  public static <I extends MessageLiteOrBuilder, O extends MessageLite.Builder> O builder(
      @Nullable I msg) {
    return msg instanceof MessageLite
        ? (O) ((MessageLite) msg).toBuilder()
        : (O) msg;
  }

  /**
   * Updates every builder from a sequence.
   *
   * @param objs List of builders to update
   * @param updater Update function. The {@code apply()} method can decide or not to update each
   *     object, and it's expected to return {@code true} if some update was made
   * @return {@code true} if at least one object was updated
   */
  public static <B extends MessageLite.Builder> boolean update(
      Iterable<B> objs, Function<B, Boolean> updater) {
    checkNotNull(updater);
    boolean updated = false;

    for (B obj : objs) {
      updated |= updater.apply(obj);
    }
    return updated;
  }

  /**
   * Runs a filter through a sequence of objects.
   *
   * @param objs Message-or-builder objects
   * @param filter Function that returns {@code true} to retain an object, {@code false} to discard
   * @return Retained objects. If some elements are retained and others are discarded,
   *     this will be a new, mutable {@link List} that contains only the retained elements.
   *     If all elements are retained, returns the same, unmodified input sequence.
   *     If all elements are discarded, returns an immutable, empty sequence
   */
  public static <M extends MessageLiteOrBuilder>
      List<M> filter(List<M> objs, Predicate<M> filter) {
    checkNotNull(filter);

    for (int i = 0; i < objs.size(); ++i) {
      if (!filter.test(objs.get(i))) {
        // At least one discarded object, go to slow-path.
        return filterFrom(objs, filter, i);
      }
    }

    // Optimized common case: all items filtered, return the input sequence.
    return objs;
  }

  private static <M extends MessageLiteOrBuilder> List<M> filterFrom(
      List<M> objs, Predicate<M> filter, int firstDiscarded) {
    List<M> filtered;

    if (firstDiscarded == 0) {
      filtered = null;
    } else {
      filtered = new ArrayList<>(objs.size() - 1);
      for (int i = 0; i < firstDiscarded; ++i) {
        filtered.add(objs.get(i));
      }
    }

    for (int i = firstDiscarded + 1; i < objs.size(); ++i) {
      M obj = objs.get(i);

      if (filter.test(obj)) {
        if (filtered == null) {
          filtered = new ArrayList<>(objs.size() - i);
        }
        filtered.add(obj);
      }
    }

    return filtered == null ? ImmutableList.<M>of() : filtered;
  }

  /**
   * Returns a copy of a {@link Message} that contains only fields that pass a filter.
   * This will be executed recursively for fields which are child messages.
   *
   * @param msg Message object
   * @param clearEmpty {@code true} will cause {@code null} to be returned if all fields from
   * {@code msg} are removed; {@code false} will return an "empty" message in that case
   * @param filter Function that returns {@code true} to retain a field, {@code false} to discard
   * @return Message with the retained fieldsfrom {@code msg}.
   *     If some fields are retained and others discarded, returns a new message object.
   *     If all fields are retained, returns the same {@code msg} object.
   *     If all fields are discarded, returns {@code null} if {@code clearEmpty==true}
   *     or a default instance of {@code msg}'s message type if {@code clearEmpty==false}
   */
  @Nullable public static <M extends Message> M filter(
      M msg, boolean clearEmpty, Predicate<FieldDescriptor> filter) {
    checkNotNull(filter);

    int i = 0;
    for (Map.Entry<FieldDescriptor, Object> entry : msg.getAllFields().entrySet()) {
      FieldDescriptor fd = entry.getKey();

      if (!filter.test(fd)) {
        // At least one field discarded, go to slow-path.
        return filterFrom(msg, clearEmpty, filter, i, true);
      } else if (fd.getType() == FieldDescriptor.Type.MESSAGE) {
        // At least one field may have children, go to slow-path.
        return filterFrom(msg, clearEmpty, filter, i, false);
      }

      ++i;
    }

    // Optimized common case: all items filtered, return the input sequence.
    return msg;
  }

  @Nullable private static <M extends Message> M filterFrom(
      M msg, boolean clearEmpty, Predicate<FieldDescriptor> filter, int first, boolean discard) {

    // At least one field may be discarded; we have to work harder and maybe create
    // a new message that will contain only the retained filters. Use a lazy-allocated
    // builder to also optimize the scenario of all fields being discarded.

    Message.Builder builder = first == 0 ? null : msg.newBuilderForType();
    Iterator<Map.Entry<FieldDescriptor, Object>> iter = msg.getAllFields().entrySet().iterator();

    for (int i = 0; i < first; ++i) {
      filterValue(clearEmpty, filter, builder, iter.next());
    }

    if (discard) {
      iter.next();
    }

    boolean updated = discard;
    while (iter.hasNext()) {
      Map.Entry<FieldDescriptor, Object> entry = iter.next();

      if (filter.test(entry.getKey())) {
        if (builder == null) {
          builder = msg.newBuilderForType();
        }
        updated |= filterValue(clearEmpty, filter, builder, entry);
      } else {
        updated = true;
      }
    }

    if (builder == null) {
      if (clearEmpty) {
        return null;
      } else {
        @SuppressWarnings("unchecked")
        M ret = (M) msg.getDefaultInstanceForType();
        return ret;
      }
    } else if (updated) {
      @SuppressWarnings("unchecked")
      M ret = (M) builder.build();
      return ret;
    } else {
      return msg;
    }
  }

  protected static boolean filterValue(boolean clearEmpty, Predicate<FieldDescriptor> filter,
      Message.Builder builder, Map.Entry<FieldDescriptor, Object> entry) {
    FieldDescriptor fd = entry.getKey();
    Object value = entry.getValue();
    boolean updated = false;

    if (fd.getType() == FieldDescriptor.Type.MESSAGE) {
      if (fd.isRepeated()) {
        for (Object obj : ((Iterable<?>) value)) {
          Message child = filter((Message) obj, clearEmpty, filter);
          if (child == null) {
            updated = true;
          } else {
            updated = child != obj;
            builder.addRepeatedField(fd, child);
          }
        }
      } else {
        Message child = filter((Message) value, clearEmpty, filter);
        if (child == null) {
          updated = true;
        } else {
          updated = child != value;
          builder.setField(fd, child);
        }
      }
    } else {
      builder.setField(fd, value);
    }

    return updated;
  }
}


/*
 * Copyright 2014 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.openrtb.json;

import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.google.protobuf.ProtocolMessageEnum;
import java.io.IOException;
import java.util.List;

/**
 * Utilities for writing JSON serialization code.
 */
public class OpenRtbJsonUtils {
  private static final long MAX_JSON_INT = 1L << 53;

  /**
   * Returns the current field name, or empty string if none.
   */
  public static String getCurrentName(JsonParser par) throws IOException {
    String name = par.getCurrentName();
    return name == null ? "" : name;
  }

  /**
   * Starts an Object, skipping the '{' token, and if necessary a field name before it.
   */
  public static void startObject(JsonParser par) throws IOException {
    JsonToken token = par.getCurrentToken();
    if (token == null || token == JsonToken.FIELD_NAME) {
      token = par.nextToken();
    }
    if (token == JsonToken.START_OBJECT) {
      par.nextToken();
    } else {
      throw new JsonParseException(par, "Expected start of object");
    }
  }

  /**
   * Returns {@code true} if the JSON Object is NOT finished.
   * Logic is inverted so this is used as a loop-end condition.
   */
  public static boolean endObject(JsonParser par) {
    JsonToken token = par.getCurrentToken();
    return token != null && token != JsonToken.END_OBJECT;
  }

  /**
   * Starts an Array, skipping the '[' token, and if necessary a field name before it.
   */
  public static void startArray(JsonParser par) throws IOException {
    JsonToken token = par.getCurrentToken();
    if (token == null || token == JsonToken.FIELD_NAME) {
      token = par.nextToken();
    }
    if (token == JsonToken.START_ARRAY) {
      par.nextToken();
    } else {
      throw new JsonParseException(par, "Expected start of array");
    }
  }

  /**
   * Returns {@code true} if the JSON Object is NOT finished.
   * Logic is inverted so this is used as a loop-end condition.
   */
  public static boolean endArray(JsonParser par) {
    JsonToken token = par.getCurrentToken();
    return token != null && token != JsonToken.END_ARRAY;
  }

  /**
   * Skips a field name if necessary, returning the current token then.
   */
  public static JsonToken peekToken(JsonParser par) throws IOException {
    JsonToken token = par.getCurrentToken();
    if (token == null || token == JsonToken.FIELD_NAME) {
      token = par.nextToken();
    }
    return token;
  }

  /**
   * Skips a field name if necessary, returning the current token then, which must be
   * the start of an Array or Object: '{' or '['.
   */
  public static JsonToken peekStructStart(JsonParser par) throws IOException {
    JsonToken token = peekToken(par);
    if (token.isStructStart()) {
      return token;
    } else {
      throw new JsonParseException(par, "Expected start of array or object");
    }
  }

  /**
   * Writes a boolean as int, where false = 0 and true = 1.
   */
  public static void writeIntBoolField(String fieldName, boolean data, JsonGenerator gen)
      throws IOException {
    gen.writeNumberField(fieldName, data ? 1 : 0);
  }

  /**
   * Writes a string array if not empty.
   */
  public static void writeStrings(String fieldName, List<String> data, JsonGenerator gen)
      throws IOException {
    if (!data.isEmpty()) {
      gen.writeArrayFieldStart(fieldName);
      for (String d : data) {
        gen.writeString(d);
      }
      gen.writeEndArray();
    }
  }

  /**
   * Writes an int array if not empty.
   */
  public static void writeInts(String fieldName, List<Integer> data, JsonGenerator gen)
      throws IOException {
    if (!data.isEmpty()) {
      gen.writeArrayFieldStart(fieldName);
      for (Integer d : data) {
        gen.writeNumber(d);
      }
      gen.writeEndArray();
    }
  }

  /**
   * Writes a long, using quotes only if it's too big (over 53-bit mantissa).
   */
  public static void writeLong(long data, JsonGenerator gen) throws IOException {
    if (data > MAX_JSON_INT || data < -MAX_JSON_INT) {
      gen.writeString(Long.toString(data));
    } else {
      gen.writeNumber(data);
    }
  }

  /**
   * Writes a long, using quotes only if it's too big (over 53-bit mantissa).
   */
  public static void writeLongField(String fieldName, long data, JsonGenerator gen)
      throws IOException {
    gen.writeFieldName(fieldName);
    writeLong(data, gen);
  }

  /**
   * Writes a long array if not empty, using quotes for values that are too big.
   *
   * @see #writeLong(long, JsonGenerator)
   */
  public static void writeLongs(String fieldName, List<Long> data, JsonGenerator gen)
      throws IOException {
    if (!data.isEmpty()) {
      gen.writeArrayFieldStart(fieldName);
      for (long d : data) {
        writeLong(d, gen);
      }
      gen.writeEndArray();
    }
  }

  /**
   * Writes a enum value as an int, using its Protobuf number.
   */
  public static void writeEnum(ProtocolMessageEnum e, JsonGenerator gen) throws IOException {
    gen.writeNumber(e.getNumber());
  }

  /**
   * Writes a enum value as an int, using its Protobuf number.
   */
  public static void writeEnumField(
      String fieldName, ProtocolMessageEnum e, JsonGenerator gen) throws IOException {
    gen.writeNumberField(fieldName, e.getNumber());
  }

  /**
   * Writes a enum array if not empty.
   *
   * @see #writeEnum(ProtocolMessageEnum, JsonGenerator)
   */
  public static void writeEnums(
      String fieldName, List<? extends ProtocolMessageEnum> enums, JsonGenerator gen)
      throws IOException {
    if (!enums.isEmpty()) {
      gen.writeArrayFieldStart(fieldName);
      for (ProtocolMessageEnum e : enums) {
        writeEnum(e, gen);
      }
      gen.writeEndArray();
    }
  }

  /**
   * Reads from either a JSON Value String (containing CSV) or a JSON Array.
   * The dual input format is needed because some fields (e.g. keywords) were allowed
   * to be of either type in OpenRTB 2.2; now in 2.3 they are all CSV strings only.
   * TODO: Simplify this to only accept CSV strings after 2.2 compatibility is dropped.
   */
  public static String readCsvString(JsonParser par) throws IOException {
    JsonToken currentToken = par.getCurrentToken();
    if (currentToken == JsonToken.START_ARRAY) {
      StringBuilder keywords = new StringBuilder();
      for (startArray(par); endArray(par); par.nextToken()) {
        if (keywords.length() != 0) {
          keywords.append(',');
        }
        keywords.append(par.getText());
      }
      return keywords.toString();
    } else if (currentToken == JsonToken.VALUE_STRING) {
      return par.getText();
    } else {
      throw new JsonParseException(par, "Expected string or array");
    }
  }
}


/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.digitalpebble.stormcrawler.util;

import java.net.IDN;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.util.Locale;
import java.util.regex.Pattern;

/** Utility class for URL analysis */
public class URLUtil {

    private URLUtil() {
    }

    /**
     * Resolve relative URL-s and fix a few java.net.URL errors in handling of
     * URLs with embedded params and pure query targets.
     *
     * @param base
     *            base url
     * @param target
     *            target url (may be relative)
     * @return resolved absolute url.
     * @throws MalformedURLException
     */
    public static URL resolveURL(URL base, String target)
            throws MalformedURLException {
        target = target.trim();

        if (target.startsWith("?")) {
            return fixPureQueryTargets(base, target);
        }

        return new URL(base, target);
    }

    /** Handle the case in RFC3986 section 5.4.1 example 7, and similar. */
    static URL fixPureQueryTargets(URL base, String target)
            throws MalformedURLException {
        if (!target.startsWith("?"))
            return new URL(base, target);

        String basePath = base.getPath();
        String baseRightMost = "";
        int baseRightMostIdx = basePath.lastIndexOf("/");
        if (baseRightMostIdx != -1) {
            baseRightMost = basePath.substring(baseRightMostIdx + 1);
        }

        if (target.startsWith("?"))
            target = baseRightMost + target;

        return new URL(base, target);
    }

    /**
     * Handles cases where the url param information is encoded into the base
     * url as opposed to the target.
     * <p>
     * If the taget contains params (i.e. ';xxxx') information then the target
     * params information is assumed to be correct and any base params
     * information is ignored. If the base contains params information but the
     * tareget does not, then the params information is moved to the target
     * allowing it to be correctly determined by the java.net.URL class.
     *
     * @param base
     *            The base URL.
     * @param target
     *            The target path from the base URL.
     *
     * @return URL A URL with the params information correctly encoded.
     *
     * @throws MalformedURLException
     *             If the url is not a well formed URL.
     */
    private static URL fixEmbeddedParams(URL base, String target)
            throws MalformedURLException {

        // the target contains params information or the base doesn't then no
        // conversion necessary, return regular URL
        if (target.indexOf(';') >= 0 || base.toString().indexOf(';') == -1) {
            return new URL(base, target);
        }

        // get the base url and it params information
        String baseURL = base.toString();
        int startParams = baseURL.indexOf(';');
        String params = baseURL.substring(startParams);

        // if the target has a query string then put the params information
        // after
        // any path but before the query string, otherwise just append to the
        // path
        int startQS = target.indexOf('?');
        if (startQS >= 0) {
            target = target.substring(0, startQS) + params
                    + target.substring(startQS);
        } else {
            target += params;
        }

        return new URL(base, target);
    }

    private static Pattern IP_PATTERN = Pattern
            .compile("(\\d{1,3}\\.){3}(\\d{1,3})");

    /** Partitions of the hostname of the url by "." */
    public static String[] getHostSegments(URL url) {
        String host = url.getHost();
        // return whole hostname, if it is an ipv4
        // TODO : handle ipv6
        if (IP_PATTERN.matcher(host).matches())
            return new String[] { host };
        return host.split("\\.");
    }

    /**
     * Partitions of the hostname of the url by "."
     *
     * @throws MalformedURLException
     */
    public static String[] getHostSegments(String url)
            throws MalformedURLException {
        return getHostSegments(new URL(url));
    }

    /**
     * Returns the lowercased hostname for the url or null if the url is not
     * well formed.
     *
     * @param url
     *            The url to check.
     * @return String The hostname for the url.
     */
    public static String getHost(String url) {
        try {
            return new URL(url).getHost().toLowerCase(Locale.ROOT);
        } catch (MalformedURLException e) {
            return null;
        }
    }

    /**
     * Returns the page for the url. The page consists of the protocol, host,
     * and path, but does not include the query string. The host is lowercased
     * but the path is not.
     *
     * @param url
     *            The url to check.
     * @return String The page for the url.
     */
    public static String getPage(String url) {
        try {
            // get the full url, and replace the query string with and empty
            // string
            url = url.toLowerCase(Locale.ROOT);
            String queryStr = new URL(url).getQuery();
            return (queryStr != null) ? url.replace("?" + queryStr, "") : url;
        } catch (MalformedURLException e) {
            return null;
        }
    }

    public static String toASCII(String url) {
        try {
            URL u = new URL(url);
            URI p = new URI(u.getProtocol(), null, IDN.toASCII(u.getHost()),
                    u.getPort(), u.getPath(), u.getQuery(), u.getRef());

            return p.toString();
        } catch (Exception e) {
            return null;
        }
    }

    public static String toUNICODE(String url) {
        try {
            URL u = new URL(url);
            URI p = new URI(u.getProtocol(), null, IDN.toUnicode(u.getHost()),
                    u.getPort(), u.getPath(), u.getQuery(), u.getRef());

            return p.toString();
        } catch (Exception e) {
            return null;
        }
    }

}


/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.digitalpebble.stormcrawler.util;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import org.apache.storm.Config;
import org.apache.storm.utils.Utils;
import org.yaml.snakeyaml.Yaml;

import clojure.lang.PersistentVector;

/* TODO replace by calls to org.apache.storm.utils.Utils */

public class ConfUtils {

    private ConfUtils() {
    }

    public static int getInt(Map<String, Object> conf, String key,
            int defaultValue) {
        Object obj = Utils.get(conf, key, defaultValue);
        return Utils.getInt(obj);
    }

    public static long getLong(Map<String, Object> conf, String key,
            long defaultValue) {
        return (Long) Utils.get(conf, key, defaultValue);
    }

    public static float getFloat(Map<String, Object> conf, String key,
            float defaultValue) {
        Object obj = Utils.get(conf, key, defaultValue);
        if (obj instanceof Double)
            return ((Double) obj).floatValue();
        return (Float) obj;
    }

    public static boolean getBoolean(Map<String, Object> conf, String key,
            boolean defaultValue) {
        Object obj = Utils.get(conf, key, defaultValue);
        return Utils.getBoolean(obj, defaultValue);
    }

    public static String getString(Map<String, Object> conf, String key) {
        return (String) Utils.get(conf, key, null);
    }

    public static String getString(Map<String, Object> conf, String key,
            String defaultValue) {
        return (String) Utils.get(conf, key, defaultValue);
    }

    /**
     * Return one or more Strings regardless of whether they are represented as
     * a single String or a list in the config or an empty List if no value
     * could be found for that key.
     **/
    public static List<String> loadListFromConf(String paramKey, Map stormConf) {
        Object obj = stormConf.get(paramKey);
        List<String> list = new LinkedList<>();

        if (obj == null)
            return list;

        if (obj instanceof PersistentVector) {
            list.addAll((PersistentVector) obj);
        } else { // single value?
            list.add(obj.toString());
        }
        return list;
    }

    public static Config loadConf(String resource, Config conf)
            throws FileNotFoundException {
        Yaml yaml = new Yaml();
        Map ret = (Map) yaml.load(new InputStreamReader(new FileInputStream(
                resource), Charset.defaultCharset()));
        if (ret == null) {
            ret = new HashMap();
        }
        // contains a single config element ?
        else {
            ret = extractConfigElement(ret);
        }
        conf.putAll(ret);
        return conf;
    }

    /**
     * If the config consists of a single key 'config', its values are used
     * instead
     **/
    public static Map extractConfigElement(Map conf) {
        if (conf.size() == 1) {
            Object confNode = conf.get("config");
            if (confNode != null && confNode instanceof Map) {
                conf = (Map) confNode;
            }
        }
        return conf;
    }
}


/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.digitalpebble.stormcrawler.protocol;

import java.net.URL;
import java.util.HashMap;

import org.apache.commons.lang.StringUtils;
import org.apache.storm.Config;

import com.digitalpebble.stormcrawler.util.ConfUtils;

public class ProtocolFactory {

    private final Config config;

    private final HashMap<String, Protocol> cache = new HashMap<>();

    public ProtocolFactory(Config conf) {
        config = conf;

        // load the list of protocols
        String[] protocols = ConfUtils.getString(conf, "protocols",
                "http,https").split(" *, *");

        // load the class names for each protocol
        // e.g. http.protocol.implementation
        for (String protocol : protocols) {
            String paramName = protocol + ".protocol.implementation";
            String protocolimplementation = ConfUtils
                    .getString(conf, paramName);
            if (StringUtils.isBlank(protocolimplementation)) {
                // set the default values
                if (protocol.equalsIgnoreCase("http")) {
                    protocolimplementation = "com.digitalpebble.stormcrawler.protocol.httpclient.HttpProtocol";
                } else if (protocol.equalsIgnoreCase("https")) {
                    protocolimplementation = "com.digitalpebble.stormcrawler.protocol.httpclient.HttpProtocol";
                } else
                    throw new RuntimeException(paramName
                            + "should not have an empty value");
            }
            // we have a value -> is it correct?
            Class protocolClass;
            try {
                protocolClass = Class.forName(protocolimplementation);
                boolean interfaceOK = Protocol.class
                        .isAssignableFrom(protocolClass);
                if (!interfaceOK) {
                    throw new RuntimeException("Class "
                            + protocolimplementation
                            + " does not implement Protocol");
                }
                Protocol protoInstance = (Protocol) protocolClass.newInstance();
                protoInstance.configure(config);
                cache.put(protocol, protoInstance);
            } catch (ClassNotFoundException e) {
                throw new RuntimeException("Can't load class "
                        + protocolimplementation);
            } catch (InstantiationException e) {
                throw new RuntimeException("Can't instanciate class "
                        + protocolimplementation);
            } catch (IllegalAccessException e) {
                throw new RuntimeException("IllegalAccessException for class "
                        + protocolimplementation);
            }
        }

    }

    public synchronized void cleanup() {
		cache.forEach((k, v) -> v.cleanup());
	}

    /** Returns an instance of the protocol to use for a given URL */
    public synchronized Protocol getProtocol(URL url) {
        // get the protocol
        String protocol = url.getProtocol();
        return cache.get(protocol);
    }
}


/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.digitalpebble.stormcrawler.bolt;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.Map;

import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;

import com.digitalpebble.stormcrawler.Metadata;
import com.digitalpebble.stormcrawler.filtering.URLFilters;
import com.digitalpebble.stormcrawler.parse.Outlink;
import com.digitalpebble.stormcrawler.persistence.Status;
import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.digitalpebble.stormcrawler.util.MetadataTransfer;
import com.digitalpebble.stormcrawler.util.URLUtil;

/**
 * Provides common functionalities for Bolts which emit tuples to the status
 * stream, e.g. Fetchers, Parsers. Encapsulates the logic of URL filtering and
 * metadata transfer to outlinks.
 **/
public abstract class StatusEmitterBolt extends BaseRichBolt {

    private URLFilters urlFilters;

    private MetadataTransfer metadataTransfer;

    private boolean allowRedirs;

    protected OutputCollector collector;

    @Override
    public void prepare(Map stormConf, TopologyContext context,
            OutputCollector collector) {
        this.collector = collector;
        urlFilters = URLFilters.fromConf(stormConf);
        metadataTransfer = MetadataTransfer.getInstance(stormConf);
        allowRedirs = ConfUtils.getBoolean(stormConf,
                com.digitalpebble.stormcrawler.Constants.AllowRedirParamName,
                true);
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declareStream(
                com.digitalpebble.stormcrawler.Constants.StatusStreamName,
                new Fields("url", "metadata", "status"));
    }

    /**
     * Used for redirections or when discovering sitemap URLs. The custom key /
     * values are added to the target metadata post-filtering.
     **/
    protected void emitOutlink(Tuple t, URL sURL, String newUrl,
            Metadata sourceMetadata, String... customKeyVals) {

        Outlink ol = filterOutlink(sURL, newUrl, sourceMetadata, customKeyVals);
        if (ol == null)
            return;

        collector.emit(
                com.digitalpebble.stormcrawler.Constants.StatusStreamName, t,
                new Values(ol.getTargetURL(), ol.getMetadata(),
                        Status.DISCOVERED));
    }

    protected Outlink filterOutlink(URL sURL, String newUrl,
            Metadata sourceMetadata, String... customKeyVals) {
        // build an absolute URL
        try {
            URL tmpURL = URLUtil.resolveURL(sURL, newUrl);
            newUrl = tmpURL.toExternalForm();
        } catch (MalformedURLException e) {
            return null;
        }

        // apply URL filters
        newUrl = this.urlFilters.filter(sURL, sourceMetadata, newUrl);

        // filtered
        if (newUrl == null) {
            return null;
        }

        Metadata metadata = metadataTransfer.getMetaForOutlink(newUrl,
                sURL.toExternalForm(), sourceMetadata);

        for (int i = 0; i < customKeyVals.length; i = i + 2) {
            metadata.addValue(customKeyVals[i], customKeyVals[i + 1]);
        }

        Outlink l = new Outlink(newUrl);
        l.setMetadata(metadata);
        return l;
    }

    protected boolean allowRedirs() {
        return allowRedirs;
    }

}