<?xml version="1.0"?>
<ownershipDocument>

    <schemaVersion>X0304</schemaVersion>

    <documentType>4</documentType>

    <periodOfReport>2011-10-21</periodOfReport>

    <notSubjectToSection16>1</notSubjectToSection16>

    <issuer>
        <issuerCik>0000810689</issuerCik>
        <issuerName>BANK OF GRANITE CORP</issuerName>
        <issuerTradingSymbol>GRAN</issuerTradingSymbol>
    </issuer>

    <reportingOwner>
        <reportingOwnerId>
            <rptOwnerCik>0001376652</rptOwnerCik>
            <rptOwnerName>Crocker Joseph D</rptOwnerName>
        </reportingOwnerId>
        <reportingOwnerAddress>
            <rptOwnerStreet1>3600 CHERRYLAUREL COURT</rptOwnerStreet1>
            <rptOwnerStreet2></rptOwnerStreet2>
            <rptOwnerCity>WINSTON SALEM</rptOwnerCity>
            <rptOwnerState>NC</rptOwnerState>
            <rptOwnerZipCode>27106</rptOwnerZipCode>
            <rptOwnerStateDescription></rptOwnerStateDescription>
        </reportingOwnerAddress>
        <reportingOwnerRelationship>
            <isDirector>1</isDirector>
            <isOfficer>0</isOfficer>
            <isTenPercentOwner>0</isTenPercentOwner>
            <isOther>0</isOther>
            <officerTitle></officerTitle>
            <otherText></otherText>
        </reportingOwnerRelationship>
    </reportingOwner>

    <nonDerivativeTable>
        <nonDerivativeTransaction>
            <securityTitle>
                <value>Common Stock</value>
            </securityTitle>
            <transactionDate>
                <value>2011-10-21</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>D</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionAmounts>
                <transactionShares>
                    <value>1007</value>
                </transactionShares>
                <transactionPricePerShare>
                    <footnoteId id="F1"/>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>D</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>0</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>D</value>
                </directOrIndirectOwnership>
            </ownershipNature>
        </nonDerivativeTransaction>
    </nonDerivativeTable>

    <derivativeTable></derivativeTable>

    <footnotes>
        <footnote id="F1">Disposed of pursuant to that certain Agreement and Plan of Merger, dated as of April 26, 2011, as amended by Amendment No. 1, dated as of June 16, 2011, and Amendment No. 2, dated as of August 15, 2011, by and among FNB United Corp., Gamma Merger Corporation and the issuer (the &quot;Merger&quot;) in exchange for 3,399 shares of FNB United Corp. common stock having a market value of $0.38 per share on the effective date of the Merger.</footnote>
    </footnotes>

    <remarks></remarks>

    <ownerSignature>
        <signatureName>/s/Kathy K. Starnes, Attorney in Fact</signatureName>
        <signatureDate>2011-10-25</signatureDate>
    </ownerSignature>
</ownershipDocument>
