+ Start a Discussion
GuyClairboisGuyClairbois 

Compression in Bulk API

Hi,

We are using the new bulk API for uploading large volumes of data with .NET Webrequests.

 

In order to improve performance, Salesforce recommends compressing the files that are sent to the systen. Does anybody know how to use compression in the outgoing webrequest? We would love to see some example .NET code, or e.g. the source code of the new DataLoader (v9), even java code would do.

 

Many thanks,

Guy

Best Answer chosen by GuyClairbois
AlwaysConfusedAlwaysConfused

Ok the Salesforce class "SForceService" is generated when you create a reference to the Enterprise WSDL.

 

from there you do this ...

 

    /// <summary>

    /// Wrapper to enable compression on the Salesforce service.

    /// </summary>

    public class GZipSforceService : SforceService

    {

        public GZipSforceService()

        {

            this.EnableDecompression = true;

        }

 

        protected override System.Net.WebRequest GetWebRequest(Uri uri)

        {

            return new GzipWebRequest(base.GetWebRequest(uri));

        }

    }

 

 

That gives you the base service class that you can use.

You then one of these to handle the compression ...

 

       /// <summary>

       /// This class wraps an existing WebRequest, and will handle compressing the request. The framework provided

    /// code will handle uncompressing the response.

       ///

       /// To use this with a Soap Client, create a new class that derives from the WSDL generated class and override GetWebRequest,

       /// its implementation should simply be

       ///           return new GzipWebRequest(base.GetWebRequest(uri));

       ///

       /// Then when using the web service, remember to create instances of the derived class, rather than the generated class

    /// This approach allows you to add compression support without having to change any generated code.

       /// </summary>

       public class GzipWebRequest : WebRequest

       {

        private WebRequest wr;

        private Stream request_stream = null;

 

              /// <summary>

              /// Construct an WebRequest wrapper that gzip compresses the request stream.

              /// </summary>

              /// <param>The WebRequest we're wrapping.</param>

              public GzipWebRequest(WebRequest wrappedRequest)

              {

                     this.wr = wrappedRequest;

                     wr.Headers["Content-Encoding"] = "gzip";

              }

 

        // most of these just delegate to the contained WebRequest

              public override string Method

              {

                     get { return wr.Method; }

                     set { wr.Method = value; }

              }

      

              public override Uri RequestUri

              {

                     get { return wr.RequestUri; }

              }

      

              public override WebHeaderCollection Headers

              {

                     get { return wr.Headers; }

                     set { wr.Headers = value; }

              }

      

              public override long ContentLength

              {

                     get { return wr.ContentLength; }

                     set { wr.ContentLength = value; }

              }

      

              public override string ContentType

              {

                     get { return wr.ContentType; }

                     set { wr.ContentType = value; }

              }

      

              public override ICredentials Credentials

              {

                     get { return wr.Credentials; }

                     set { wr.Credentials = value; }

              }

      

              public override bool PreAuthenticate

              {

                     get { return wr.PreAuthenticate; }

                     set { wr.PreAuthenticate = value; }

              }

      

              public override System.IO.Stream GetRequestStream()

              {

                     return WrappedRequestStream(wr.GetRequestStream());

              }

      

              public override IAsyncResult BeginGetRequestStream(AsyncCallback callback, object state)

              {

                     return wr.BeginGetRequestStream (callback, state);

              }

      

              public override System.IO.Stream EndGetRequestStream(IAsyncResult asyncResult)

              {

                     return WrappedRequestStream(wr.EndGetRequestStream(asyncResult));

              }

      

              /// <summary>

              /// helper function that wraps the request stream in a GZipStream

              /// </summary>

              /// <param>The Stream we're compressing</param>

              /// <returns></returns>

              private Stream WrappedRequestStream(Stream requestStream)

              {

                     if ( request_stream == null )

                           request_stream = new GZipStream(requestStream, CompressionMode.Compress);

                     return request_stream;

              }

 

        public override WebResponse GetResponse()

        {

            try

            {

                return wr.GetResponse();

            }

            catch (Exception e)

            {

                throw e;

            }

        }

      

              public override IAsyncResult BeginGetResponse(AsyncCallback callback, object state)

              {

                     return wr.BeginGetResponse (callback, state);

              }

      

              public override WebResponse EndGetResponse(IAsyncResult asyncResult)

              {

                     return wr.EndGetResponse(asyncResult);

              }

       }

}

All Answers

AlwaysConfusedAlwaysConfused

Ok the Salesforce class "SForceService" is generated when you create a reference to the Enterprise WSDL.

 

from there you do this ...

 

    /// <summary>

    /// Wrapper to enable compression on the Salesforce service.

    /// </summary>

    public class GZipSforceService : SforceService

    {

        public GZipSforceService()

        {

            this.EnableDecompression = true;

        }

 

        protected override System.Net.WebRequest GetWebRequest(Uri uri)

        {

            return new GzipWebRequest(base.GetWebRequest(uri));

        }

    }

 

 

That gives you the base service class that you can use.

You then one of these to handle the compression ...

 

       /// <summary>

       /// This class wraps an existing WebRequest, and will handle compressing the request. The framework provided

    /// code will handle uncompressing the response.

       ///

       /// To use this with a Soap Client, create a new class that derives from the WSDL generated class and override GetWebRequest,

       /// its implementation should simply be

       ///           return new GzipWebRequest(base.GetWebRequest(uri));

       ///

       /// Then when using the web service, remember to create instances of the derived class, rather than the generated class

    /// This approach allows you to add compression support without having to change any generated code.

       /// </summary>

       public class GzipWebRequest : WebRequest

       {

        private WebRequest wr;

        private Stream request_stream = null;

 

              /// <summary>

              /// Construct an WebRequest wrapper that gzip compresses the request stream.

              /// </summary>

              /// <param>The WebRequest we're wrapping.</param>

              public GzipWebRequest(WebRequest wrappedRequest)

              {

                     this.wr = wrappedRequest;

                     wr.Headers["Content-Encoding"] = "gzip";

              }

 

        // most of these just delegate to the contained WebRequest

              public override string Method

              {

                     get { return wr.Method; }

                     set { wr.Method = value; }

              }

      

              public override Uri RequestUri

              {

                     get { return wr.RequestUri; }

              }

      

              public override WebHeaderCollection Headers

              {

                     get { return wr.Headers; }

                     set { wr.Headers = value; }

              }

      

              public override long ContentLength

              {

                     get { return wr.ContentLength; }

                     set { wr.ContentLength = value; }

              }

      

              public override string ContentType

              {

                     get { return wr.ContentType; }

                     set { wr.ContentType = value; }

              }

      

              public override ICredentials Credentials

              {

                     get { return wr.Credentials; }

                     set { wr.Credentials = value; }

              }

      

              public override bool PreAuthenticate

              {

                     get { return wr.PreAuthenticate; }

                     set { wr.PreAuthenticate = value; }

              }

      

              public override System.IO.Stream GetRequestStream()

              {

                     return WrappedRequestStream(wr.GetRequestStream());

              }

      

              public override IAsyncResult BeginGetRequestStream(AsyncCallback callback, object state)

              {

                     return wr.BeginGetRequestStream (callback, state);

              }

      

              public override System.IO.Stream EndGetRequestStream(IAsyncResult asyncResult)

              {

                     return WrappedRequestStream(wr.EndGetRequestStream(asyncResult));

              }

      

              /// <summary>

              /// helper function that wraps the request stream in a GZipStream

              /// </summary>

              /// <param>The Stream we're compressing</param>

              /// <returns></returns>

              private Stream WrappedRequestStream(Stream requestStream)

              {

                     if ( request_stream == null )

                           request_stream = new GZipStream(requestStream, CompressionMode.Compress);

                     return request_stream;

              }

 

        public override WebResponse GetResponse()

        {

            try

            {

                return wr.GetResponse();

            }

            catch (Exception e)

            {

                throw e;

            }

        }

      

              public override IAsyncResult BeginGetResponse(AsyncCallback callback, object state)

              {

                     return wr.BeginGetResponse (callback, state);

              }

      

              public override WebResponse EndGetResponse(IAsyncResult asyncResult)

              {

                     return wr.EndGetResponse(asyncResult);

              }

       }

}

This was selected as the best answer
AlwaysConfusedAlwaysConfused

I use that for handling communication in both directions not just uploading.

 

I create batches and other such stuff in an api wrapper that I built around these 2 classes.

 

Initially i had some performance issues, but I figured those out ...

http://community.salesforce.com/t5/NET-Development/API-Performance-seems-extremely-low-when-compared-to-the-data/m-p/186605#M8236

 

My justification for needing this is here ...

http://community.salesforce.com/t5/NET-Development/Syncronisation/m-p/186599#M8233

 

So ... I think i covered everything :)

 

Let me know if i missed anything.