How to Use Selenium with PowerShell [Tutorial 2025]

$Driver = Start-SeChrome -WebDriverDirectory '/path/to/chromedriver/folder' -Headless

Import-Module -Name Selenium

# Initialize Selenium WebDriver
$Driver = Start-SeChrome -WebDriverDirectory './chromedriver-win64' -Headless

# Visit G2 reviews page
Enter-SeUrl 'https://www.g2.com/products/airtable/reviews' -Driver $Driver

# Wait a moment for page to load
Start-Sleep -Seconds 5

# Get the page title
$Title = $Driver.Title
Write-Host "Page title: $Title"

# Print the page source
$Html = $Driver.PageSource
$Html | Out-File -FilePath "g2_response.html"

# Close the browser
Stop-SeDriver -Driver $Driver

<div class="paper__bd">
  <div class="d-f mb-1">
    <div class="f-1 d-f ai-c mb-half-small-only">
      <div class="stars large xlarge--medium-down stars-10"></div>
      <div class="time-stamp pl-4th ws-nw">
        <span><span class="x-current-review-date" data-poison-date="">
          <meta content="2025-02-11" itemprop="datePublished">
          <time datetime="2025-02-11">Feb 11, 2025</time>
        </span></span>
      </div>
    </div>
  </div>
  <div class="d-f">
    <div class="f-1">
      <div data-poison="">
        <a class="pjax" href="https://www.g2.com/products/airtable/reviews/airtable-review-10822745">
          <div class="m-0 l2" itemprop="name">"Helped us get a handle on our complicated team-based scheduling and management."</div>
        </a>
      </div>
      <div data-poison-text="">
        <div itemprop="reviewBody">
          <div>
            <div class="l5 mt-2">What do you like best about Airtable?</div>
            <div>
              <p class="formatted-text">The best part about airtable is it's customiizable nature. We worked with a tech specialist to configure Airtable to meet our unique needs and it has so far been adaptable to everything we needed it to do...</p>
            </div>
          </div>
          <!-- More review sections... -->
        </div>
      </div>
    </div>
  </div>
</div>

node --version
npm --version

npm install puppeteer-core

// scrape.js
const puppeteer = require('puppeteer-core');

async function scrapeG2Reviews() {
  // Replace with your actual Scrapeless API token
  const API_TOKEN = 'YOUR_API_TOKEN_HERE';
  
  // Configure connection to Scrapeless Scraping Browser
  const connectionURL = `wss://browser.scrapeless.com/browser?token=${API_TOKEN}&session_ttl=180&proxy_country=ANY`;
  
  console.log('Connecting to Scrapeless Scraping Browser...');
  
  const browser = await puppeteer.connect({
    browserWSEndpoint: connectionURL,
    defaultViewport: null,
  });
  
  console.log('Connected!');
  
  const page = await browser.newPage();
  
  // Set a longer timeout for navigation
  page.setDefaultNavigationTimeout(120000);
  
  // Array to hold all reviews
  const allReviews = [];
  
  // Number of pages to scrape
  const pagesToScrape = 3;
  
  for (let currentPage = 1; currentPage <= pagesToScrape; currentPage++) {
    console.log(`Navigating to page ${currentPage}...`);
    
    // Navigate directly to the specific page number
    const pageUrl = currentPage === 1 
      ? 'https://www.g2.com/products/airtable/reviews' 
      : `https://www.g2.com/products/airtable/reviews?page=${currentPage}`;
      
    await page.goto(pageUrl, {
      waitUntil: 'networkidle2',
      timeout: 60000
    });
    
    console.log(`Page ${currentPage} loaded. Taking screenshot...`);
    
    // Take a screenshot to verify the page loaded correctly
    await page.screenshot({ path: `g2_page_${currentPage}.png` });
    
    // Wait for the page content to load
    // Try different possible selectors since the structure might have changed
    try {
      console.log('Waiting for content to load...');
      
      // First try the original selector
      await page.waitForSelector('.paper__bd', { timeout: 10000 })
        .then(() => console.log('Found selector: .paper__bd'))
        .catch(() => console.log('Selector .paper__bd not found, trying alternatives...'));
      
      // If we're still here, let's check the page structure using another approach
      console.log('Analyzing page structure...');
      
      // Get the page title to verify we're on the right page
      const pageTitle = await page.title();
      console.log('Page title:', pageTitle);
      
      // Extract reviews with a more generic approach
      console.log('Extracting reviews...');
      
      // Try to extract reviews with a more resilient approach
      const pageReviews = await page.evaluate(() => {
        // Look for review containers with multiple selectors
        const reviewElements = Array.from(
          document.querySelectorAll('.paper__bd, .review-item, .reviews-section article, [data-testid="review-card"]')
        );
        
        console.log(`Found ${reviewElements.length} potential review elements`);
        
        if (reviewElements.length === 0) {
          // If no reviews found with specific selectors, let's dump the page structure
          const html = document.documentElement.outerHTML;
          return { error: 'No reviews found', html: html.substring(0, 5000) }; // First 5000 chars for debugging
        }
        
        return reviewElements.map(review => {
          // Try to get title with multiple potential selectors
          const titleElement = 
            review.querySelector('[itemprop="name"], .review-title, h3, h4') ||
            review.querySelector('a.pjax div');
            
          const title = titleElement ? titleElement.innerText.trim() : 'No title';
          
          // Try to get date with multiple potential selectors
          const dateElement = 
            review.querySelector('time, .review-date, [data-testid="review-date"]') ||
            review.querySelector('.time-stamp');
            
          const date = dateElement ? dateElement.innerText.trim() : 'No date';
          
          // Try to get rating with multiple approaches
          let rating = 0;
          const starsElement = review.querySelector('.stars, .rating, [data-testid="star-rating"]');
          
          if (starsElement) {
            // Check for a stars-X class
            const starsClass = starsElement.className;
            const starsMatch = starsClass.match(/stars-(\d+)/);
            if (starsMatch) {
              rating = parseInt(starsMatch[1]) / 2;
            } else {
              // Otherwise check for aria-label that might contain the rating
              const ariaLabel = starsElement.getAttribute('aria-label');
              if (ariaLabel) {
                const ratingMatch = ariaLabel.match(/(\d+(\.\d+)?)/);
                if (ratingMatch) {
                  rating = parseFloat(ratingMatch[1]);
                }
              }
            }
          }
          
          // Try to get review text with multiple potential selectors
          const textElements = review.querySelectorAll('.formatted-text, .review-content, p, [data-testid="review-content"]');
          let content = '';
          
          textElements.forEach(el => {
            const text = el.innerText.trim();
            if (text && !text.includes('Review collected by and hosted on G2.com')) {
              content += text + '\n\n';
            }
          });
          
          return {
            title,
            date,
            rating,
            content: content.trim() || 'No content found'
          };
        });
      });
      
      if (pageReviews.error) {
        console.log('Error extracting reviews:', pageReviews.error);
        console.log('Page HTML snippet:', pageReviews.html);
      } else {
        console.log(`Extracted ${pageReviews.length} reviews from page ${currentPage}`);
        
        // Add to our collection
        allReviews.push(...pageReviews);
      }
      
    } catch (error) {
      console.error('Error on page:', error);
      // Take another screenshot on error for debugging
      await page.screenshot({ path: `error_page_${currentPage}.png` });
      
      // Dump page HTML for debugging
      const html = await page.content();
      require('fs').writeFileSync(`error_page_${currentPage}.html`, html);
      console.log(`Saved error page HTML to error_page_${currentPage}.html`);
    }
    
    // Small pause between pages to be nice to the server
    await new Promise(r => setTimeout(r, 2000));
  }
  
  console.log(`Total reviews extracted: ${allReviews.length}`);
  
  // Save all reviews to a file
  require('fs').writeFileSync('g2_reviews.json', JSON.stringify(allReviews, null, 2));
  console.log('Reviews saved to g2_reviews.json');
  
  await browser.close();
  console.log('Done!');
}

scrapeG2Reviews().catch(console.error);

node scrape.js

[
  {
    "title": "\"Helped us get a handle on our complicated team-based scheduling and management.\"",
    "date": "Feb 11, 2025",
    "rating": 5,
    "content": "What do you like best about Airtable?\nThe best part about airtable is it's customiizable nature. We worked with a tech specialist to configure Airtable to meet our unique needs and it has so far been adaptable to everything we needed it to do. It has a huge capacity in terms of features and we were able to get it up and running in a fairly short period of time.\n\nWhat do you dislike about Airtable?\nThere are no real downsides except the intial learning curve. I believe we could have built Airtable out as we needed but it would have been a bigger intial time investment. Working with a tech specialist who knew Airtable well really helped us get up and running fast. It was worth the cost."
  },
  {
    "title": "\"Great for organization, has changed our workflow\"",
    "date": "Feb 8, 2025",
    "rating": 4.5,
    "content": "What do you like best about Airtable?\nWhat I like best about Airtable is its flexibility and customizability. It allows us to create databases that perfectly fit our needs, with custom fields, views, and formulas. The ability to link records between tables is incredibly powerful.\n\nWhat do you dislike about Airtable?\nSometimes the learning curve can be steep for new users, especially when dealing with more complex formulas and automations. The pricing can also become quite expensive as your needs grow and you require more records or features."
  },
  {
    "title": "\"Powerful database tool with a spreadsheet interface\"",
    "date": "Jan 29, 2025",
    "rating": 4,
    "content": "What do you like best about Airtable?\nAirtable combines the familiarity of spreadsheets with the power of a database. I appreciate how it allows for different views of the same data - grid, kanban, calendar, etc. It's intuitive enough for non-technical users but powerful enough for complex data management.\n\nWhat do you dislike about Airtable?\nThe free tier is quite limited, and pricing jumps significantly for premium features. Sometimes performance can slow down with large datasets, and there are occasional sync issues when multiple team members are editing simultaneously."
  }
]

# File: run-scraper.ps1

Write-Host "Starting G2.com scraping with Scrapeless Scraping Browser..."

# Create output folder
$outputFolder = "G2_Reviews_" + (Get-Date -Format "yyyyMMdd_HHmmss")
New-Item -ItemType Directory -Path $outputFolder -Force | Out-Null

# Run the Node.js script and capture the output
$jsonFile = Join-Path $outputFolder "reviews.json"
node scrape.js > $jsonFile

Write-Host "Node.js scraping complete. Processing results..."

# Check if the file contains valid JSON data
try {
    # Read the file content
    $content = Get-Content -Path $jsonFile -Raw
    
    # Extract the JSON part (assuming it's the output after all console.log messages)
    if ($content -match '\[.*\]') {
        $jsonContent = $Matches[0]
        Set-Content -Path $jsonFile -Value $jsonContent
        
        # Parse the JSON to PowerShell objects
        $reviews = $jsonContent | ConvertFrom-Json
        
        # Export to CSV
        $csvPath = Join-Path $outputFolder "g2_reviews.csv"
        $reviews | Export-Csv -Path $csvPath -NoTypeInformation -Encoding UTF8
        
        Write-Host "Successfully extracted $($reviews.Count) reviews"
        Write-Host "Results saved to: $csvPath"
        
        # Basic analysis
        $averageRating = ($reviews | Measure-Object -Property rating -Average).Average
        Write-Host "Average rating: $averageRating out of 5"
        
        $ratingDistribution = $reviews | Group-Object -Property rating | Sort-Object -Property Name
        foreach ($ratingGroup in $ratingDistribution) {
            Write-Host "$($ratingGroup.Name) stars: $($ratingGroup.Count) reviews"
        }
    } else {
        Write-Error "Could not find JSON data in the output"
    }
} catch {
    Write-Error "Error processing the JSON data: $_"
}

.\run-scraper.ps1

// File: scrape-g2-multipage.js
const puppeteer = require('puppeteer-core');

async function scrapeG2ReviewsMultiPage() {
  // Replace with your actual Scrapeless API token
  const API_TOKEN = 'YOUR_API_TOKEN_HERE';
  
  // Configure connection to Scrapeless Scraping Browser
  const connectionURL = `wss://browser.scrapeless.com/browser?token=${API_TOKEN}&session_ttl=300&proxy_country=ANY`;
  
  console.log('Connecting to Scrapeless Scraping Browser...');
  
  const browser = await puppeteer.connect({
    browserWSEndpoint: connectionURL,
    defaultViewport: null,
  });
  
  console.log('Connected! Opening G2.com...');
  
  const page = await browser.newPage();
  
  // Set a reasonable navigation timeout
  page.setDefaultNavigationTimeout(60000);
  
  // Array to hold all reviews
  const allReviews = [];
  
  // Function to extract reviews from current page
  const extractReviews = async () => {
    return page.evaluate(() => {
      return Array.from(document.querySelectorAll('.paper__bd')).map(review => {
        // Title
        const titleElement = review.querySelector('a.pjax div[itemprop="name"]');
        const title = titleElement ? titleElement.innerText : 'No title';
        
        // Date
        const dateElement = review.querySelector('time');
        const date = dateElement ? dateElement.innerText : 'No date';
        
        // Rating
        const starsElement = review.querySelector('.stars');
        let rating = 0;
        if (starsElement) {
          const starsClass = starsElement.className;
          const match = starsClass.match(/stars-(\d+)/);
          if (match) {
            rating = parseInt(match[1])/2;
          }
        }
        
        // Review content
        const contentElements = review.querySelectorAll('.formatted-text');
        let content = '';
        contentElements.forEach(el => {
          const text = el.innerText;
          if (!text.includes('Review collected by and hosted on G2.com')) {
            content += text + '\n\n';
          }
        });
        
        return {
          title,
          date,
          rating,
          content: content.trim()
        };
      });
    });
  };
  
  // Navigate to the first page
  await page.goto('https://www.g2.com/products/airtable/reviews', {
    waitUntil: 'networkidle2'
  });
  
  // Wait for reviews to load
  await page.waitForSelector('.paper__bd', {timeout: 30000});
  
  // Take a screenshot of the first page
  await page.screenshot({ path: 'page1.png' });
  
  // Number of pages to scrape (limit for demonstration)
  const pagesToScrape = 3;
  let currentPage = 1;
  
  while (currentPage <= pagesToScrape) {
    console.log(`Scraping page ${currentPage}...`);
    
    // Extract reviews from current page
    const pageReviews = await extractReviews();
    console.log(`Found ${pageReviews.length} reviews on page ${currentPage}`);
    
    // Add to our collection
    allReviews.push(...pageReviews);
    
    // Check if there's a next page button
    const hasNextPage = await page.evaluate(() => {
      const nextButton = document.querySelector('a.next_page');
      return nextButton && !nextButton.classList.contains('disabled');
    });
    
    if (hasNextPage && currentPage < pagesToScrape) {
      // Navigate directly to the next page URL
      currentPage++;
      await page.goto(`https://www.g2.com/products/airtable/reviews?page=${currentPage}`, {
        waitUntil: 'networkidle2'
      });
      
      // Wait for the reviews to load on the new page
      await page.waitForSelector('.paper__bd', {timeout: 30000});
      
      // Take a screenshot of the new page
      await page.screenshot({ path: `page${currentPage}.png` });
    } else {
      break;
    }
  }
  
  console.log(`Total reviews extracted: ${allReviews.length}`);
  
  // Output all reviews as JSON
  console.log(JSON.stringify(allReviews, null, 2));
  
  await browser.close();
  console.log('Done!');
}

scrapeG2ReviewsMultiPage().catch(console.error);

npm install puppeteer-core

const puppeteer = require('puppeteer-core');

(async () => {
  // Replace with your actual Scrapeless API token
  const API_TOKEN = 'YOUR_API_TOKEN_HERE';
  
  const connectionURL = `wss://browser.scrapeless.com/browser?token=${API_TOKEN}&session_ttl=180&proxy_country=ANY`;
  
  console.log('Connecting to Scrapeless Scraping Browser...');
  
  const browser = await puppeteer.connect({
    browserWSEndpoint: connectionURL,
    defaultViewport: null,
  });
  
  console.log('Connected!');
  
  const page = await browser.newPage();
  await page.goto('https://www.example.com', {
    waitUntil: 'networkidle2',
    timeout: 60000
  });
  
  console.log('Page loaded successfully!');
  console.log('Page title:', await page.title());
  
  // Take a screenshot
  await page.screenshot({ path: 'screenshot.png' });
  
  // Your scraping logic here
  // ...
  
  await browser.close();
  console.log('Done!');
})().catch(console.error);

node scrape.js