There’s a customer I was working with who had a restricted network and was trying to determine exactly why a piece of software was failing. In their environment, they had a proxy server setup that would let them ‘poke holes’ in the firewall to let specific URLs through to the machine. Their challenge was figuring out exactly which URLs they need to configure at the proxy to enable their software to work.
In order for me to help, I needed a way to simulate their environment in a way that I could gather more diagnostic information and have control over the end to end. The good news is that tools & technology exist already to make this almost work! I could setup a machine in Azure and install Fiddler to monitor all the network traffic. (Fiddler is awesome, check it out if you aren’t familiar! with it already!) The missing link to make this test environment work together was the ability to selectively enable URLs and block the rest.
The good news is that Fiddler has a rich extensibility model so we can just extend Fiddler with the additional functionality – great!
I wrote a Fiddler .Net Extension that will read in a text file (one URL per line) and if the destination URL of the request matches the URL, then Fiddler allows it through. If it doesn’t match, Fiddler will block the request. This lets me do fine grained URL Whitelisting! In addition, with the log that Fiddler provides, it’s very clear what URLs get blocked so I can copy/paste, add to the whitelist, refresh and try again until the scenario is working properly. It’s a great way to determine what URLs need to be whitelisted to make it work!
To configure the Test Environment
1. Enable Fiddler
- Install Fiddler
- Run Fiddler at least once, this creates the <%USERPROFILE%\Documents\Fiddler2\Scripts> directory
- Copy an already built version of the code below (as a DLL) to the Scripts folder
- Create a text file named Whitelist.txt in the Scripts folder (this is where we put the URLs to whitelist)
- Add a URL like http://www.bing.com in the Whitelist.txt file
2. Configure Windows Firewall to block everything except Fiddler
- Open Windows Firewall with Advanced Security
- Click “Windows Firewall Properties”
- Choose “Block” for Outbound Connections for any active profiles and click “OK”
- Right-Click “Outbound Rules” in the left pane and click “New Rule”
- On the first tab (Rule Type), choose “Program”
- On the second tab (Program), navigate to where you installed Fiddler, for example: “C:\Program Files (x86)\Fiddler2\Fiddler.exe”
- On the third tab (Action), choose “Allow the connection”
- On the fourth tab (Profile), leave all the boxes checked
- On the fifth tab (Name), choose a name and description for the rule and click “OK”
Let’s Try out the Test Environment
Now that you have a test environment configured, let’s try it out! Launch Fiddler, choose “FiddlerWhitelist” on the menu and click the first menu item to Turn ON Whitelist Blocking. Once this is turned on (checked), only URLs that ‘match’ what’s in the Whitelist.txt file will be allowed and everything else blocked! You can see in the screenshot below, http://a4.bing.com was blocked, when http://www.bing.com was allowed!
Here’s the code for the Fiddler Extension:
//
// Prototype content-blocker allows testing of websites when some resources are blocked.
//
using System;
using System.Collections;
using System.Globalization;
using System.Collections.Generic;
using System.Windows.Forms;
using System.Text;
using Fiddler;
using System.IO;
using System.Diagnostics;
using Microsoft.Win32;
using System.Reflection;
[assembly: Fiddler.RequiredVersion("2.4.9.0")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]
[assembly: AssemblyTitle("FiddlerWhitelist")]
[assembly: AssemblyDescription("Block ALL requests except whitelist")]
[assembly: AssemblyProduct("FiddlerWhitelist")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyCopyright("Copyright © 2015")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
public class FiddlerWhitelist : IAutoTamper, IHandleExecAction
{
private List EnabledHosts;
private bool bBlockerEnabled = false;
string sSecret = new Random().Next().ToString();
private System.Windows.Forms.MenuItem mnuFiddlerWhitelist;
private System.Windows.Forms.MenuItem miFiddlerWhitelistEnabled;
private System.Windows.Forms.MenuItem miReadWhitelist;
private string WhitelistLocation;
private string LogLocation;
private void InitializeMenu()
{
mnuFiddlerWhitelist = new System.Windows.Forms.MenuItem();
miFiddlerWhitelistEnabled = new System.Windows.Forms.MenuItem();
miReadWhitelist = new System.Windows.Forms.MenuItem();
//
// mnuContentBlock
//
mnuFiddlerWhitelist.MenuItems.AddRange(new System.Windows.Forms.MenuItem[] { miFiddlerWhitelistEnabled,
miReadWhitelist});
mnuFiddlerWhitelist.Text = "&FiddlerWhitelist";
//
// miContentBlockEnabled
//
miFiddlerWhitelistEnabled.Index = 0;
miFiddlerWhitelistEnabled.Text = "&Turn ON Whitelist Blocking";
miFiddlerWhitelistEnabled.Click += new System.EventHandler(miBlockRule_Click);
miReadWhitelist.Index = 1;
miReadWhitelist.Text = "&Refresh URLs From Whitelist.txt";
miReadWhitelist.Click += new System.EventHandler(miRefreshURLs_Click);
}
///
/// Ensures a 1x1 Transparent GIF file is in the \Responses\ subfolder.
/// This image will be returned by Fiddler for certain blocked content.
///
private void EnsureTransGif()
{
if (!File.Exists(CONFIG.GetPath("Responses") + "1pxtrans.dat"))
{
try
{
byte[] arrHeaders = Encoding.UTF8.GetBytes("HTTP/1.1 404 Blocked\r\nContentBlock: True\r\nDate: Wed, 31 Oct 2012 16:41:35 GMT\r\nContent-Type: image/gif\r\nConnection: close\r\nContent-Length: 49\r\n\r\n");
byte[] arrBody = { 0x47, 0x49, 0x46, 0x38, 0x39, 0x61, 0x01, 0x00, 0x01, 0x00, 0x91, 0xFF, 0x00,
0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0xC0, 0xC0, 0xC0, 0x00, 0x00, 0x00, 0x21,
0xF9, 0x04, 0x01, 0x00, 0x00, 0x02, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x00, 0x01,
0x00, 0x01, 0x00, 0x00, 0x02, 0x02, 0x54, 0x01, 0x00, 0x3B
};
FileStream oFS = File.Create(CONFIG.GetPath("Responses") + "1pxtrans.dat");
oFS.Write(arrHeaders, 0, arrHeaders.Length);
oFS.Write(arrBody, 0, arrBody.Length);
oFS.Close();
}
catch (Exception eX)
{
MessageBox.Show(eX.ToString(), "Failed to create transparent gif...");
}
}
}
///
///
///
///
private void miBlockRule_Click(object sender, System.EventArgs e)
{
MenuItem oSender = (sender as MenuItem);
oSender.Checked = !oSender.Checked;
bBlockerEnabled = miFiddlerWhitelistEnabled.Checked;
if (bBlockerEnabled)
{
//File.AppendAllText(LogLocation, Environment.NewLine + "Whitelisting turned on at: " + DateTime.Now.ToString() + Environment.NewLine);
miFiddlerWhitelistEnabled.Text = "&Turn OFF Whitelist Blocking";
EnsureTransGif();
}
else
{
miFiddlerWhitelistEnabled.Text = "&Turn ON Whitelist Blocking";
}
}
private void miRefreshURLs_Click(object sender, System.EventArgs e)
{
MenuItem oSender = (sender as MenuItem);
ReadURLs();
}
public FiddlerWhitelist()
{
WhitelistLocation = Path.Combine(Path.GetDirectoryName(Assembly.GetAssembly(typeof(FiddlerWhitelist)).Location), "Whitelist.txt");
LogLocation = Path.Combine(Path.GetDirectoryName(Assembly.GetAssembly(typeof(FiddlerWhitelist)).Location), "Log.txt");
ReadURLs();
InitializeMenu();
}
private void ReadURLs()
{
// we need to read in a file that contains the whitelist
EnabledHosts = new List();
// Read in hosts from file and populate the host list - ideally this would be configurable in the UI
if (File.Exists(WhitelistLocation))
{
string[] urls = File.ReadAllLines(WhitelistLocation);
foreach (string s in urls)
{
if (!string.IsNullOrWhiteSpace(s))
{
EnabledHosts.Add(s.ToLower().Trim());
}
}
}
else
{
MessageBox.Show("Unable to find Whitelist.txt file here: " + WhitelistLocation);
}
}
public void OnLoad()
{
/*
* NB: You might not get called here until ~after~ one of the IAutoTamper methods was called.
* This is okay for us, because we created our mnuContentBlock in the constructor and its simply not
* visible anywhere until this method is called and we merge it onto the Fiddler Main menu.
*/
FiddlerApplication.UI.mnuMain.MenuItems.Add(mnuFiddlerWhitelist);
}
public void OnBeforeUnload()
{
}
///
/// IHandleExecAction handler.
/// Respond to user input from QuickExec box under the Web Sessions list...
///
/// The Command String
/// TRUE if our extension has handled the command.
public bool OnExecAction(string sCommand)
{
//// TODO: Add "BLOCKSITE" and "UNBLOCKSITE" commands
//if (String.Equals(sCommand, "blocklist", StringComparison.OrdinalIgnoreCase))
//{
// MessageBox.Show(EnabledHosts.ToString(), "Block List...");
// return true;
//}
return false;
}
///
/// This function kills known matches early
///
///
public void AutoTamperRequestBefore(Session oSession)
{
// Return immediately if Content Blocking is disabled
if (!bBlockerEnabled) return;
string fullUrl = oSession.fullUrl.ToLower();
//File.AppendAllText(LogLocation, fullUrl + Environment.NewLine);
bool foundMatch = false;
for (int i = 0; i < EnabledHosts.Count && !foundMatch; i++)
{
if (fullUrl.Contains(EnabledHosts[i]))
{
foundMatch = true;
}
}
if (!foundMatch)
{
oSession.oRequest.FailSession(404, "Fiddler - Whitelist",
String.Format("This site was blocked by Fiddler Whitelist", oSession.fullUrl, sSecret));
}
}
public void AutoTamperResponseAfter(Session oSession) { /*noop*/ }
public void AutoTamperRequestAfter(Session oSession) { /*noop*/ }
public void AutoTamperResponseBefore(Session oSession) {/*noop*/}
public void OnBeforeReturningError(Session oSession) {/*noop*/}
public void OnAfterSessionComplete(Session oSession) { /*noop*/ }
// Change class to implement IAutoTamper to IAutoTamper2 to add this callback
// public void OnPeekAtResponseHeaders(Session oSession) { /*noop*/ }
// Change class to implement IAutoTamper to IAutoTamper3 to add this callback
// public void OnPeekAtResponseHeaders(Session oSession) { /*noop*/ }
}