Friday, October 15, 2010

using asp.net and Amazon S3 buckets to save files from your website\application

So you have a website, and u want your users to be able to upload and download files, why not use Amazons S3 buckets to store the files? it will help u save space on your server.
I presume you know what S3 buckets is, but if not, click here to find out more and get your own buckets.
There is a few S3 classes that you will need, download it here

First you would want to get a list of all your buckets and the files in the buckets

Private Sub ListS3Buckets()
        Dim s3B As AmazonS3REST.AWSAuthConnection = New AmazonS3REST.AWSAuthConnection(PublicKey, SecretKey) 'use the publickey and secret key you got from amazon
        Dim aBucketList As ArrayList
        aBucketList = s3B.ListAllMyBuckets(Nothing).Buckets 'gets a arraylist of all the buckets

        For Each s3Buckets As AmazonS3REST.Bucket In aBucketList

            Dim sBucketName As String = s3Buckets.Name


            Dim cList As ArrayList 'get a list of al the items in the buckets
            cList = s3B.ListBucket(s3Buckets.Name, Nothing, Nothing, 0, Nothing).Entries ' gets a arraylist of all the objects in the bucket
            For Each cN As AmazonS3REST.ListEntry In cList
                Dim sItemName As String = cN.Key
                If cN.Size = 0 Then
                    'if the size is zero then this is a sub dir
                    'add your code here   
                Else
                    'this item is a file
                    'add your code here
                End If

            Next

        Next

    End Sub


To Create a bucket

Private Sub CreateS3Buckets()
        Dim s3B As AmazonS3REST.AWSAuthConnection = New AmazonS3REST.AWSAuthConnection(PublicKey, SecretKey)
        s3B.CreateBucket(txtNewBucketName.Text, Nothing)

    End Sub

To upload a file to the bucket, in this example we first save the file to a temp dir on the server, then upload it to the S3 bucket, after the upload the file is deleted from the temp dir.
Protected Sub Button1_Click(ByVal sender As Object, ByVal e As System.EventArgs) Handles Upload.Click

        Dim sBucketName As String
        sBucketName = "your bucket name" 'get the selected bucket name, this is the selected treenode

        'if u want the file in a sub dir append the bucket name with the subdir name
        'uncomment the next line to add subdir
        'sBucketName = sBucketName + "/" + "subdir name here"

        If SaveUploadFileToTempDir() Then ' check to see if the files is saved in the temp dir (upload)
           
                FileThumbName = ThumbNameFromFileName(UploadFile.FileName)


                PushToAmazonS3ViaREST(sBucketName, UploadFile.FileName) 'upload the file to the S3 storage
           
                DeleteLocalFile(UploadPhysicalPath, UploadFile.FileName)

        End If

    End Sub

    ''' 
    ''' Upload the file to the S3 storage
    ''' 
    ''' The bucket name to upload the file to''' the file name to upload''' 
    Private Sub PushToAmazonS3ViaREST(ByVal Bucket As String, ByVal FileNameToUpload As String)
        Dim s3 As AmazonS3REST.AWSAuthConnection = New AmazonS3REST.AWSAuthConnection(PublicKey, SecretKey)
        Dim sContentType As String = UploadFile.PostedFile.ContentType.ToString()
        Dim sList As SortedList = New SortedList()
        sList.Add("Content-Type", sContentType)
        sList.Add("x-amz-acl", "public-read")
        Dim obj As AmazonS3REST.S3Object = New AmazonS3REST.S3Object(FileContentsAsString(UploadPhysicalPath + FileNameToUpload), sList)
        s3.PutObjectAsStream(Bucket, FileNameToUpload, UploadPhysicalPath + FileNameToUpload, obj.Metadata)
        s3 = Nothing

    End Sub

'delete the temp file on the server
   Private Sub DeleteLocalFile(ByVal LocalDir As String, ByVal FileName As String)
        File.Delete(LocalDir + FileName)
    End Sub

'upload and save the file to server
    Private Function SaveUploadFileToTempDir() As Boolean
        Dim fileType As String = UploadFile.PostedFile.ContentType.ToString()
        Dim bResult As Boolean = False
        If UploadFile.PostedFile IsNot Nothing Then
            
                ' If file exists, it will be replaced
                UploadFile.PostedFile.SaveAs(UploadPhysicalPath + UploadFile.FileName)
                UploadFile.Dispose()
                Message.Text = ""
                bResult = True
       
        End If
        Return bResult
    End Function

'stream the file
  Public Function FileContentsAsString(ByVal path As String) As String
        'Given a file name, return its contents as a data stream
        Dim fs As FileStream = New System.IO.FileStream(path, System.IO.FileMode.Open, System.IO.FileAccess.Read)
        Dim binaryData As [Byte]() = New [Byte](fs.Length - 1) {}
        Dim offset As Integer = 0
        Dim remaining As Integer = CInt(fs.Length)
        While remaining > 0
            Dim read As Int32 = fs.Read(binaryData, offset, remaining)
            If read <= 0 Then
                Throw New EndOfStreamException([String].Format("End of stream reached. Aborted with {0} bytes left to read", remaining))
            End If
            remaining -= read
            offset += read
        End While
        Dim enc As New System.Text.ASCIIEncoding()
        fs.Close()
        fs.Dispose()
        Dim str As String = enc.GetString(binaryData)
        Return str
    End Function



That is it, to delete a file is easy

''' 
    ''' Delete a file in a bucket object = file
    ''' 
    ''' the bucket name of the object to delete''' the object name to delete''' 
    Private Sub DeleteS3BucketObject(ByVal sBucketName As String, ByVal sObject As String)
        Dim s3B As AmazonS3REST.AWSAuthConnection = New AmazonS3REST.AWSAuthConnection(PublicKey, SecretKey)
        s3B.DeleteObject(sBucketName, sObject, Nothing)
    End Sub

And finally to delete a bucket
''' 
    ''' Delete the bucket
    ''' 
    ''' 
    Private Sub DeleteS3Bucket()
        Dim s3B As AmazonS3REST.AWSAuthConnection = New AmazonS3REST.AWSAuthConnection(PublicKey, SecretKey)
        s3B.DeleteBucket(Session("sSelectedBucket"), Nothing)
    End Sub

Thats it, enjoy your buckets.
(I found most of this code somewhere on the interweb, all credit to the authors)