Nodejs get data from url

I'm trying to read the content from a URL with Node.js but all I seem to get are a bunch of bytes. I'm obviously doing something wrong but I'm not sure what. This is the code I currently have:

var http = require('http');

var client = http.createClient(80, "google.com");
request = client.request();
request.on('response', function( res ) {
    res.on('data', function( data ) {
        console.log( data );
    } );
} );
request.end();

Any insight would be greatly appreciated.

asked Jun 9, 2011 at 2:02

try using the on error event of the client to find the issue.

var http = require('http');

var options = {
    host: 'google.com',
    path: '/'
}
var request = http.request(options, function (res) {
    var data = '';
    res.on('data', function (chunk) {
        data += chunk;
    });
    res.on('end', function () {
        console.log(data);

    });
});
request.on('error', function (e) {
    console.log(e.message);
});
request.end();

answered Jun 20, 2012 at 9:10

user896993user896993

1,30112 silver badges15 bronze badges

3

HTTP and HTTPS:

const getScript = (url) => {
    return new Promise((resolve, reject) => {
        const http      = require('http'),
              https     = require('https');

        let client = http;

        if (url.toString().indexOf("https") === 0) {
            client = https;
        }

        client.get(url, (resp) => {
            let data = '';

            // A chunk of data has been recieved.
            resp.on('data', (chunk) => {
                data += chunk;
            });

            // The whole response has been received. Print out the result.
            resp.on('end', () => {
                resolve(data);
            });

        }).on("error", (err) => {
            reject(err);
        });
    });
};

(async (url) => {
    console.log(await getScript(url));
})('https://sidanmor.com/');

answered Feb 11, 2018 at 8:29

sidanmorsidanmor

4,8633 gold badges21 silver badges31 bronze badges

the data object is a buffer of bytes. Simply call .toString() to get human-readable code:

console.log( data.toString() );

reference: Node.js buffers

mooreds

4,7362 gold badges31 silver badges37 bronze badges

answered Jun 9, 2011 at 2:07

1

A slightly modified version of @sidanmor 's code. The main point is, not every webpage is purely ASCII, user should be able to handle the decoding manually (even encode into base64)

function httpGet(url) {
  return new Promise((resolve, reject) => {
    const http = require('http'),
      https = require('https');

    let client = http;

    if (url.toString().indexOf("https") === 0) {
      client = https;
    }

    client.get(url, (resp) => {
      let chunks = [];

      // A chunk of data has been recieved.
      resp.on('data', (chunk) => {
        chunks.push(chunk);
      });

      // The whole response has been received. Print out the result.
      resp.on('end', () => {
        resolve(Buffer.concat(chunks));
      });

    }).on("error", (err) => {
      reject(err);
    });
  });
}

(async(url) => {
  var buf = await httpGet(url);
  console.log(buf.toString('utf-8'));
})('https://httpbin.org/headers');

answered Sep 24, 2020 at 3:14

ttimasdfttimasdf

1,2691 gold badge11 silver badges18 bronze badges

Not the answer you're looking for? Browse other questions tagged node.js or ask your own question.

How does Node read data from a website?

How do you access a URL in Node?

How do I get Node.js data?

How do I get Node text from a website?