<?xml version="1.0"?>
<ownershipDocument>

    <schemaVersion>X0202</schemaVersion>

    <documentType>4</documentType>

    <periodOfReport>2006-03-29</periodOfReport>

    <notSubjectToSection16>0</notSubjectToSection16>

    <issuer>
        <issuerCik>0000925464</issuerCik>
        <issuerName>SECURITY BANK CORP</issuerName>
        <issuerTradingSymbol>SBKC</issuerTradingSymbol>
    </issuer>

    <reportingOwner>
        <reportingOwnerId>
            <rptOwnerCik>0001194053</rptOwnerCik>
            <rptOwnerName>RAMSEY JOHN W</rptOwnerName>
        </reportingOwnerId>
        <reportingOwnerAddress>
            <rptOwnerStreet1>8401 SAXONY</rptOwnerStreet1>
            <rptOwnerStreet2></rptOwnerStreet2>
            <rptOwnerCity>MACON</rptOwnerCity>
            <rptOwnerState>GA</rptOwnerState>
            <rptOwnerZipCode>31220</rptOwnerZipCode>
            <rptOwnerStateDescription></rptOwnerStateDescription>
        </reportingOwnerAddress>
        <reportingOwnerRelationship>
            <isDirector>1</isDirector>
            <isOfficer>0</isOfficer>
            <isTenPercentOwner>0</isTenPercentOwner>
            <isOther>0</isOther>
            <officerTitle></officerTitle>
            <otherText></otherText>
        </reportingOwnerRelationship>
    </reportingOwner>

    <nonDerivativeTable>
        <nonDerivativeTransaction>
            <securityTitle>
                <value>COMMON STOCK</value>
            </securityTitle>
            <transactionDate>
                <value>2006-03-29</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>J</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionTimeliness>
                <value></value>
            </transactionTimeliness>
            <transactionAmounts>
                <transactionShares>
                    <value>9892</value>
                    <footnoteId id="F1"/>
                </transactionShares>
                <transactionPricePerShare>
                    <value>22.85</value>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>A</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>550170</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>I</value>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value>100% OWNER OF GFSE PARTNERSHIP</value>
                </natureOfOwnership>
            </ownershipNature>
        </nonDerivativeTransaction>
    </nonDerivativeTable>

    <footnotes>
        <footnote id="F1">THESE SHARES ARE ISSUED PURSUANT TO THE ASSET PURCHASE AGREEMENT BETWEEN SECURITY BANK CORPORATION, FORMALLY KNOWN AS SNB BANKCHSRES, INC., AND FAIRFIELD FINANCIAL SERVICES, INC., DATED JULY 31, 2000, IN WHICH SECURITY BANK CORPORATION ISSUES SHARES OF ITS' COMMON STOCK TO GROUP FINANICAL SOUTHEAST (GFSE), A WHOLLY OWNED PARTNERSHIP BY MR. RAMSEY, OVER A FIVE YEAR PERIOD CONTINGENT UPON FIARFIELD'S PERFORMANCE.</footnote>
    </footnotes>

    <remarks></remarks>

    <ownerSignature>
        <signatureName>/s/ John W. Ramsey</signatureName>
        <signatureDate>2006-03-29</signatureDate>
    </ownerSignature>
</ownershipDocument>
