1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

module AsynchCrawl

open System.Net
open System
open System.IO
open System.Text.RegularExpressions
open Microsoft.FSharp.Control.WebExtensions
open Microsoft.FSharp.Control

let httpAsync (<A>url:string</A>) = 
    async { 
             try
                let req = WebRequest.Create(url)    
                let! resp = req.AsyncGetResponse()  //Todo get errors and timeout.

                let stream = resp.GetResponseStream()
                let reader = new StreamReader(stream)
  
                let contents = reader.ReadToEnd()      // Todo make Async           let! contents = reader.AsyncReadToEnd()
                return contents  // return (url, html)

            with e ->
                 return sprintf "An Exception happened: %s" e.Message 
    
        }  

let parallelWorker n f =
    MailboxProcessor.Start( fun inbox ->
        let workers = 
            Array.init n ( fun i -> MailboxProcessor.Start(f))
        let rec loop i = async {
            let! msg = inbox.Receive()
            workers.[i].Post(msg)
            return! loop ((i+1) %n)
        }
        loop 0
    )


let linkPat = "href=\s*\"[^\"h]*(<A href="http://[%5e&/%22]*)/">http://[^&\"]*)\</A>""
let getLinks (txt:string) =
    [ for m in Regex.Matches(txt,linkPat) -> m.Groups.Item(1).Value ]


let collectLinks (<A>url:string</A>) =
    async { // An Async web request with a global gate

        let! html =  httpAsync(url) 
        let links = getLinks html

        do printfn "finished reading %s, got %d links" 
                url (List.length links)

        return links }

let urlCollector  (parallelcount:int) = 
    parallelWorker parallelcount (fun self -> 
        let rec waitForUrl (visited : Set<string>) =
            async { if visited.Count < 10 then
                         let! url = self.Receive()
                         if not (visited.Contains(url)) then
                              Async.Start 
                                (async { let! links = collectLinks url
                                         for link in links do
                                             do self <-- link })
                         return! waitForUrl(visited.Add(url)) }

        waitForUrl(Set.empty))

urlCollector(8) <-- "<A href="http://news.google.com/">http://news.google.com</A>"
System.Console.Read() |> ignore